-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathlog.txt
646 lines (605 loc) · 56.9 KB
/
log.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
/home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
_np_qint8 = np.dtype([("qint8", np.int8, 1)])
/home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
_np_quint8 = np.dtype([("quint8", np.uint8, 1)])
/home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
_np_qint16 = np.dtype([("qint16", np.int16, 1)])
/home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
_np_quint16 = np.dtype([("quint16", np.uint16, 1)])
/home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
_np_qint32 = np.dtype([("qint32", np.int32, 1)])
/home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
np_resource = np.dtype([("resource", np.ubyte, 1)])
INFO:absl:[32m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>[0m
WARNING:absl:[33m1. Before generating data : Memory used is 241.953125[0m
INFO:absl:[32m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<[0m
0%| | 0/5 [00:00<?, ?it/s] 20%|██ | 1/5 [00:04<00:17, 4.43s/it] 40%|████ | 2/5 [00:08<00:13, 4.44s/it] 60%|██████ | 3/5 [00:13<00:08, 4.46s/it] 80%|████████ | 4/5 [00:17<00:04, 4.47s/it]100%|██████████| 5/5 [00:22<00:00, 4.48s/it]100%|██████████| 5/5 [00:22<00:00, 4.48s/it]
0%| | 0/2 [00:00<?, ?it/s] 50%|█████ | 1/2 [00:04<00:04, 4.61s/it]100%|██████████| 2/2 [00:09<00:00, 4.54s/it]100%|██████████| 2/2 [00:09<00:00, 4.50s/it]
INFO:absl:[32m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>[0m
WARNING:absl:[33m2. Before defining model : Memory used is 300.515625[0m
INFO:absl:[32m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<[0m
/home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/memory_profiler.py:649: UserWarning: Could not extract a code object for the object <class '__main__.NNet'>
% func)
INFO:absl:[32m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>[0m
WARNING:absl:[33m3. Before defining estimator : Memory used is 300.515625[0m
INFO:absl:[32m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<[0m
Writing to data/train_data/0.tfrecords
Writing to data/train_data/1.tfrecords
Writing to data/train_data/2.tfrecords
Writing to data/train_data/3.tfrecords
Writing to data/train_data/4.tfrecords
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
114 242.0 MiB 242.0 MiB @profile
115 def generate_numpy_tf_records(out_dir,
116 num_tfrecord_files=5,
117 num_samples_per_file=100000,
118 num_features=250):
119 """
120 Generates random data for Linear Regression and stores them as TFRecords
121 :param num_tfrecord_files: Number of TF records
122 :param out_dir: Out directory path
123 :return:
124 """
125 242.0 MiB 0.0 MiB if not os.path.exists(out_dir):
126 242.0 MiB 0.0 MiB os.makedirs(out_dir)
127
128 300.4 MiB 0.2 MiB for i in tqdm(range(num_tfrecord_files)):
129 300.0 MiB 0.0 MiB file_path_name = os.path.join(out_dir, str(i) + ".tfrecords") # ~ 106MB
130 300.0 MiB 0.0 MiB print(f"Writing to {file_path_name}")
131 300.0 MiB 0.0 MiB if os.path.exists(file_path_name):
132 print(f"Found : {file_path_name}")
133 else:
134 # generate regression dataset
135 300.2 MiB 38.7 MiB X, Y = make_regression(n_samples=num_samples_per_file, n_features=num_features, noise=0.1)
136
137 # plot regression dataset
138
139 300.2 MiB 0.1 MiB with tf.io.TFRecordWriter(file_path_name) as writer:
140 300.2 MiB 0.0 MiB for x, y in zip(X, Y):
141 # (2500000 * 8) / 1024 /1024 ~ 19MB ~ 10MB on disk
142 # (10000 * 8) / 1024 /1024 ~ 0.076 MB
143 # get_numpy_array_size(features)
144 # get_numpy_array_size(label)
145 #create TF Features
146 300.2 MiB 0.0 MiB features = tf.train.Features(feature=_get_regression_features(data=x, label=y))
147 # create TF Example
148 300.2 MiB 0.0 MiB example = tf.train.Example(features=features)
149 # print(example)
150 300.2 MiB 0.0 MiB writer.write(example.SerializeToString())
Writing to data/val_data/0.tfrecords
Writing to data/val_data/1.tfrecords
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
114 262.2 MiB 262.2 MiB @profile
115 def generate_numpy_tf_records(out_dir,
116 num_tfrecord_files=5,
117 num_samples_per_file=100000,
118 num_features=250):
119 """
120 Generates random data for Linear Regression and stores them as TFRecords
121 :param num_tfrecord_files: Number of TF records
122 :param out_dir: Out directory path
123 :return:
124 """
125 262.2 MiB 0.0 MiB if not os.path.exists(out_dir):
126 262.2 MiB 0.0 MiB os.makedirs(out_dir)
127
128 300.5 MiB 0.0 MiB for i in tqdm(range(num_tfrecord_files)):
129 281.3 MiB 0.0 MiB file_path_name = os.path.join(out_dir, str(i) + ".tfrecords") # ~ 106MB
130 281.3 MiB 0.0 MiB print(f"Writing to {file_path_name}")
131 281.3 MiB 0.0 MiB if os.path.exists(file_path_name):
132 print(f"Found : {file_path_name}")
133 else:
134 # generate regression dataset
135 300.5 MiB 19.2 MiB X, Y = make_regression(n_samples=num_samples_per_file, n_features=num_features, noise=0.1)
136
137 # plot regression dataset
138
139 300.5 MiB 0.0 MiB with tf.io.TFRecordWriter(file_path_name) as writer:
140 300.5 MiB 0.0 MiB for x, y in zip(X, Y):
141 # (2500000 * 8) / 1024 /1024 ~ 19MB ~ 10MB on disk
142 # (10000 * 8) / 1024 /1024 ~ 0.076 MB
143 # get_numpy_array_size(features)
144 # get_numpy_array_size(label)
145 #create TF Features
146 300.5 MiB 0.0 MiB features = tf.train.Features(feature=_get_regression_features(data=x, label=y))
147 # create TF Example
148 300.5 MiB 0.0 MiB example = tf.train.Example(features=features)
149 # print(example)
150 300.5 MiB 0.0 MiB writer.write(example.SerializeToString())
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
301 300.5 MiB 300.5 MiB @profile
302 def _init_tf_config(total_steps_per_file,
303 model_dir,
304 clear_model_data=False,
305 keep_checkpoint_max=5):
306
307 300.5 MiB 0.0 MiB save_checkpoints_steps= total_steps_per_file * 2
308 # each TFRecord file has NUM_SAMPLE, so for every 2 TFRecord files store the checkpoint
309
310 300.5 MiB 0.0 MiB save_summary_steps= total_steps_per_file / 5 # log 5 times per file
311 300.5 MiB 0.0 MiB log_step_count_steps= total_steps_per_file / 5
312
313 300.7 MiB 0.2 MiB run_config = tf.compat.v1.ConfigProto()
314 300.7 MiB 0.0 MiB run_config.gpu_options.allow_growth = True
315 # run_config.gpu_options.per_process_gpu_memory_fraction = 0.50
316 300.7 MiB 0.0 MiB run_config.allow_soft_placement = True
317 300.7 MiB 0.0 MiB run_config.log_device_placement = False
318 300.7 MiB 0.0 MiB model_dir = model_dir
319
320 300.7 MiB 0.0 MiB if clear_model_data:
321 if os.path.exists(model_dir):
322 shutil.rmtree(model_dir)
323
324 300.7 MiB 0.0 MiB _run_config = tf.estimator.RunConfig(session_config=run_config,
WARNING:tensorflow:Estimator's model_fn (<__main__.NNet object at 0x7f8c72c4ffd0>) includes params argument, but params are not passed to Estimator.
WARNING:tensorflow:Estimator's model_fn (<__main__.NNet object at 0x7f8c72c4ffd0>) includes params argument, but params are not passed to Estimator.
INFO:absl:[32m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>[0m
WARNING:absl:[33m4. Before training : Memory used is 300.734375[0m
INFO:absl:[32m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<[0m
tfrecords size: 0%| | 0/5 [00:00<?, ?it/s]2019-09-07 11:55:36.591836: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1
2019-09-07 11:55:36.607838: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:55:36.608342: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties:
name: GeForce GTX 1060 with Max-Q Design major: 6 minor: 1 memoryClockRate(GHz): 1.48
pciBusID: 0000:01:00.0
2019-09-07 11:55:36.608488: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0
2019-09-07 11:55:36.609271: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2019-09-07 11:55:36.609924: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0
2019-09-07 11:55:36.610089: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0
2019-09-07 11:55:36.610878: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0
2019-09-07 11:55:36.611465: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0
2019-09-07 11:55:36.613336: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2019-09-07 11:55:36.613416: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:55:36.613949: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:55:36.614307: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0
2019-09-07 11:55:36.651603: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
2019-09-07 11:55:36.672845: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2208000000 Hz
2019-09-07 11:55:36.673949: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x56001b94e8b0 executing computations on platform Host. Devices:
2019-09-07 11:55:36.673963: I tensorflow/compiler/xla/service/service.cc:175] StreamExecutor device (0): Host, Default Version
2019-09-07 11:55:36.728781: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:55:36.729584: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x56001b7e65a0 executing computations on platform CUDA. Devices:
2019-09-07 11:55:36.729599: I tensorflow/compiler/xla/service/service.cc:175] StreamExecutor device (0): GeForce GTX 1060 with Max-Q Design, Compute Capability 6.1
2019-09-07 11:55:36.729727: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:55:36.729966: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties:
name: GeForce GTX 1060 with Max-Q Design major: 6 minor: 1 memoryClockRate(GHz): 1.48
pciBusID: 0000:01:00.0
2019-09-07 11:55:36.729995: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0
2019-09-07 11:55:36.730005: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2019-09-07 11:55:36.730014: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0
2019-09-07 11:55:36.730024: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0
2019-09-07 11:55:36.730032: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0
2019-09-07 11:55:36.730044: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0
2019-09-07 11:55:36.730054: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2019-09-07 11:55:36.730093: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:55:36.730337: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:55:36.730554: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0
2019-09-07 11:55:36.730576: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0
2019-09-07 11:55:36.731174: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-09-07 11:55:36.731184: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165] 0
2019-09-07 11:55:36.731188: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0: N
2019-09-07 11:55:36.731250: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:55:36.731505: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:55:36.731744: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 5292 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1060 with Max-Q Design, pci bus id: 0000:01:00.0, compute capability: 6.1)
tfrecords size: 20%|██ | 1/5 [00:06<00:24, 6.06s/it]tfrecords size: 40%|████ | 2/5 [00:11<00:18, 6.01s/it]tfrecords size: 60%|██████ | 3/5 [00:18<00:12, 6.10s/it]tfrecords size: 80%|████████ | 4/5 [00:24<00:06, 6.13s/it]tfrecords size: 100%|██████████| 5/5 [00:30<00:00, 6.13s/it]tfrecords size: 100%|██████████| 5/5 [00:30<00:00, 6.12s/it]
tfrecords size: 0%| | 0/2 [00:00<?, ?it/s]tfrecords size: 50%|█████ | 1/2 [00:05<00:05, 5.73s/it]tfrecords size: 100%|██████████| 2/2 [00:11<00:00, 5.74s/it]tfrecords size: 100%|██████████| 2/2 [00:11<00:00, 5.75s/it]
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_core/python/training/training_util.py:236: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.
Instructions for updating:
Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.
WARNING:tensorflow:Entity <bound method CodeMap.items of {<code object numpy_array_decode at 0x7f8ce2c89390, file "test_memory_leak.py", line 152>: {}}> appears to be a generator function. It will not be converted by AutoGraph.
WARNING:tensorflow:Entity <bound method CodeMap.items of {<code object numpy_array_decode at 0x7f8ce2c89390, file "test_memory_leak.py", line 152>: {}}> appears to be a generator function. It will not be converted by AutoGraph.
INFO:absl:[32m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>[0m
WARNING:absl:[33mDefining model... : Memory used is 473.8984375[0m
INFO:absl:[32m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<[0m
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/model_fn.py:337: scalar (from tensorflow.python.framework.tensor_shape) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.TensorShape([]).
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/model_fn.py:337: scalar (from tensorflow.python.framework.tensor_shape) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.TensorShape([]).
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py:1486: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py:1486: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
2019-09-07 11:56:48.562444: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:56:48.562770: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties:
name: GeForce GTX 1060 with Max-Q Design major: 6 minor: 1 memoryClockRate(GHz): 1.48
pciBusID: 0000:01:00.0
2019-09-07 11:56:48.562813: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0
2019-09-07 11:56:48.562833: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2019-09-07 11:56:48.562853: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0
2019-09-07 11:56:48.562871: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0
2019-09-07 11:56:48.562889: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0
2019-09-07 11:56:48.562906: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0
2019-09-07 11:56:48.562925: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2019-09-07 11:56:48.562981: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:56:48.563262: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:56:48.563506: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0
2019-09-07 11:56:48.563529: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-09-07 11:56:48.563537: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165] 0
2019-09-07 11:56:48.563543: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0: N
2019-09-07 11:56:48.563607: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:56:48.563886: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:56:48.564175: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 5292 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1060 with Max-Q Design, pci bus id: 0000:01:00.0, compute capability: 6.1)
2019-09-07 11:57:39.978363: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
WARNING:tensorflow:Entity <bound method CodeMap.items of {<code object numpy_array_decode at 0x7f8ce2c89390, file "test_memory_leak.py", line 152>: {}}> appears to be a generator function. It will not be converted by AutoGraph.
WARNING:tensorflow:Entity <bound method CodeMap.items of {<code object numpy_array_decode at 0x7f8ce2c89390, file "test_memory_leak.py", line 152>: {}}> appears to be a generator function. It will not be converted by AutoGraph.
325 300.7 MiB 0.0 MiB save_checkpoints_steps=save_checkpoints_steps,
326 300.7 MiB 0.0 MiB keep_checkpoint_max=keep_checkpoint_max,
327 300.7 MiB 0.0 MiB save_summary_steps=save_summary_steps,
328 300.7 MiB 0.0 MiB model_dir=model_dir,
329 300.7 MiB 0.0 MiB log_step_count_steps=log_step_count_steps)
330
331 300.7 MiB 0.0 MiB return _run_config
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
210 300.7 MiB 300.7 MiB @profile
211 def get_tf_records_count(path):
212 300.7 MiB 0.0 MiB path = os.path.join(path, "*.tfrecords").replace("//", "/")
213 300.7 MiB 0.0 MiB files = glob.glob(path)
214 300.7 MiB 0.0 MiB total_records = -1
215 420.7 MiB 0.0 MiB for file in tqdm(files, desc="tfrecords size: "):
216 # total_records += sum(1 for _ in tf.python_io.tf_record_iterator(file))
217 420.7 MiB 115.4 MiB total_records += sum(1 for _ in tf.data.TFRecordDataset(file))
218 420.7 MiB 0.0 MiB return total_records
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
333 300.7 MiB 300.7 MiB @profile
334 def _get_train_spec(train_data_path, batch_size, num_features, num_epochs=None, max_steps=None):
335 # Estimators expect an input_fn to take no arguments.
336 # To work around this restriction, we use lambda to capture the arguments and provide the expected interface.
337 420.7 MiB 119.9 MiB _total_num_samples = get_tf_records_count(train_data_path)
338 420.7 MiB 0.0 MiB steps_per_epoch = _total_num_samples // batch_size
339
340 420.7 MiB 0.0 MiB if max_steps is None:
341 420.7 MiB 0.0 MiB max_steps = steps_per_epoch * num_epochs
342
343 420.7 MiB 0.0 MiB return tf.estimator.TrainSpec(
344 420.7 MiB 0.0 MiB input_fn=lambda: _get_dataset(data_path=train_data_path,
345 batch_size=batch_size,
346 num_features=num_features),
347 420.7 MiB 0.0 MiB max_steps=max_steps,
348 420.7 MiB 0.0 MiB hooks=None)
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
210 420.7 MiB 420.7 MiB @profile
211 def get_tf_records_count(path):
212 420.7 MiB 0.0 MiB path = os.path.join(path, "*.tfrecords").replace("//", "/")
213 420.7 MiB 0.0 MiB files = glob.glob(path)
214 420.7 MiB 0.0 MiB total_records = -1
215 420.7 MiB 0.0 MiB for file in tqdm(files, desc="tfrecords size: "):
216 # total_records += sum(1 for _ in tf.python_io.tf_record_iterator(file))
217 420.7 MiB 0.0 MiB total_records += sum(1 for _ in tf.data.TFRecordDataset(file))
218 420.7 MiB 0.0 MiB return total_records
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
350 420.7 MiB 420.7 MiB @profile
351 def _get_eval_spec(val_data_path, batch_size, num_features, num_epochs=None, max_steps=None):
352
353 420.7 MiB 0.0 MiB _total_num_samples = get_tf_records_count(val_data_path)
354 420.7 MiB 0.0 MiB STEPS_PER_EPOCH = _total_num_samples // batch_size
355
356 420.7 MiB 0.0 MiB if max_steps is None:
357 420.7 MiB 0.0 MiB max_steps = STEPS_PER_EPOCH
358
359 420.7 MiB 0.0 MiB return tf.estimator.EvalSpec(
360 420.7 MiB 0.0 MiB input_fn=lambda: _get_dataset(data_path=val_data_path,
361 batch_size=batch_size,
362 num_features=num_features),
363 420.7 MiB 0.0 MiB steps=max_steps,
364 420.7 MiB 0.0 MiB hooks=None)
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
173 421.0 MiB 421.0 MiB @profile
174 def _get_dataset(data_path,
175 batch_size,
176 num_features):
177 """
178 Reads TFRecords, decode and batches them
179 :return: dataset
180 """
181 421.0 MiB 0.0 MiB _num_cores = 4
182
183 421.0 MiB 0.0 MiB path = os.path.join(data_path, "*.tfrecords")
184 421.0 MiB 0.0 MiB path = path.replace("//", "/")
185 421.3 MiB 0.3 MiB files = tf.data.Dataset.list_files(path)
186 # files = glob.glob(pathname=path)
187
188 # TF dataset APIs
189 421.3 MiB 0.0 MiB dataset = files.interleave(
190 421.3 MiB 0.0 MiB tf.data.TFRecordDataset,
191 421.3 MiB 0.0 MiB cycle_length=_num_cores,
192 421.5 MiB 0.2 MiB num_parallel_calls=tf.data.experimental.AUTOTUNE)
193
194 # dataset = tf.data.TFRecordDataset(files, num_parallel_reads=_num_cores)
195 # dataset = dataset.shuffle(_batch_size*10, 42)
196 # Map the generator output as features as a dict and label
197
198 421.5 MiB 0.0 MiB dataset = dataset.map(map_func=lambda serialized_example : numpy_array_decode(serialized_example=serialized_example,
199 num_features=num_features),
200 473.9 MiB 52.4 MiB num_parallel_calls=tf.data.experimental.AUTOTUNE)
201 473.9 MiB 0.0 MiB dataset = dataset.batch(batch_size=batch_size, drop_remainder=False)
202 473.9 MiB 0.0 MiB dataset = dataset.repeat()
203
204 # dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
205 # iterator = dataset.make_one_shot_iterator()
206 # batch_feats, batch_label = iterator.get_next()
207 # return batch_feats, batch_label
208 473.9 MiB 0.0 MiB return dataset
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
173 5766.7 MiB 5766.7 MiB @profile
174 def _get_dataset(data_path,
175 batch_size,
176 num_features):
177 """
178 Reads TFRecords, decode and batches them
INFO:absl:[32m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>[0m
WARNING:absl:[33mDefining model... : Memory used is 5816.48828125[0m
INFO:absl:[32m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<[0m
2019-09-07 11:57:57.020096: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:57:57.020328: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties:
name: GeForce GTX 1060 with Max-Q Design major: 6 minor: 1 memoryClockRate(GHz): 1.48
pciBusID: 0000:01:00.0
2019-09-07 11:57:57.020371: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0
2019-09-07 11:57:57.020390: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2019-09-07 11:57:57.020407: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0
2019-09-07 11:57:57.020424: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0
2019-09-07 11:57:57.020440: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0
2019-09-07 11:57:57.020477: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0
2019-09-07 11:57:57.020493: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2019-09-07 11:57:57.020541: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:57:57.020739: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:57:57.020909: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0
2019-09-07 11:57:57.020931: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-09-07 11:57:57.020938: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165] 0
2019-09-07 11:57:57.020944: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0: N
2019-09-07 11:57:57.021002: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:57:57.021171: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 11:57:57.021319: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 5292 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1060 with Max-Q Design, pci bus id: 0000:01:00.0, compute capability: 6.1)
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_core/python/training/saver.py:963: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to delete files with this prefix.
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_core/python/training/saver.py:963: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.
Instructions for updating:
Use standard file APIs to delete files with this prefix.
WARNING:tensorflow:Entity <bound method CodeMap.items of {<code object numpy_array_decode at 0x7f8ce2c89390, file "test_memory_leak.py", line 152>: {}}> appears to be a generator function. It will not be converted by AutoGraph.
WARNING:tensorflow:Entity <bound method CodeMap.items of {<code object numpy_array_decode at 0x7f8ce2c89390, file "test_memory_leak.py", line 152>: {}}> appears to be a generator function. It will not be converted by AutoGraph.
INFO:absl:[32m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>[0m
WARNING:absl:[33mDefining model... : Memory used is 10215.41015625[0m
INFO:absl:[32m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<[0m
2019-09-07 12:00:00.881171: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 12:00:00.881370: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties:
name: GeForce GTX 1060 with Max-Q Design major: 6 minor: 1 memoryClockRate(GHz): 1.48
pciBusID: 0000:01:00.0
2019-09-07 12:00:00.881402: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0
2019-09-07 12:00:00.881413: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2019-09-07 12:00:00.881422: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0
2019-09-07 12:00:00.881430: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0
2019-09-07 12:00:00.881438: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0
2019-09-07 12:00:00.881447: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0
2019-09-07 12:00:00.881455: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2019-09-07 12:00:00.881498: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 12:00:00.881671: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 12:00:00.881822: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0
2019-09-07 12:00:00.881842: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-09-07 12:00:00.881847: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165] 0
2019-09-07 12:00:00.881851: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0: N
2019-09-07 12:00:00.881905: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 12:00:00.882079: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 12:00:00.882230: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 5292 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1060 with Max-Q Design, pci bus id: 0000:01:00.0, compute capability: 6.1)
INFO:absl:[32m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>[0m
WARNING:absl:[33m5. Before exporitng the model : Memory used is 10256.05859375[0m
INFO:absl:[32m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<[0m
INFO:absl:Saving model to =======> data/model/exported/
INFO:absl:[32m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>[0m
WARNING:absl:[33mDefining model... : Memory used is 10256.0390625[0m
INFO:absl:[32m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<[0m
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_core/python/saved_model/signature_def_utils_impl.py:201: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.
WARNING:tensorflow:From /home/mageswarand/anaconda3/envs/tie/lib/python3.6/site-packages/tensorflow_core/python/saved_model/signature_def_utils_impl.py:201: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.
2019-09-07 12:00:12.053658: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 12:00:12.053871: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties:
name: GeForce GTX 1060 with Max-Q Design major: 6 minor: 1 memoryClockRate(GHz): 1.48
pciBusID: 0000:01:00.0
2019-09-07 12:00:12.053902: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0
2019-09-07 12:00:12.053913: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0
2019-09-07 12:00:12.053922: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0
2019-09-07 12:00:12.053931: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0
2019-09-07 12:00:12.053939: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0
2019-09-07 12:00:12.053948: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0
2019-09-07 12:00:12.053958: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7
2019-09-07 12:00:12.053997: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 12:00:12.054173: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 12:00:12.054317: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0
2019-09-07 12:00:12.054337: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:
2019-09-07 12:00:12.054342: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165] 0
2019-09-07 12:00:12.054346: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0: N
2019-09-07 12:00:12.054400: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 12:00:12.054580: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2019-09-07 12:00:12.054744: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 5292 MB memory) -> physical GPU (device: 0, name: GeForce GTX 1060 with Max-Q Design, pci bus id: 0000:01:00.0, compute capability: 6.1)
179 :return: dataset
180 """
181 5766.7 MiB 0.0 MiB _num_cores = 4
182
183 5766.7 MiB 0.0 MiB path = os.path.join(data_path, "*.tfrecords")
184 5766.7 MiB 0.0 MiB path = path.replace("//", "/")
185 5766.7 MiB 0.0 MiB files = tf.data.Dataset.list_files(path)
186 # files = glob.glob(pathname=path)
187
188 # TF dataset APIs
189 5766.7 MiB 0.0 MiB dataset = files.interleave(
190 5766.7 MiB 0.0 MiB tf.data.TFRecordDataset,
191 5766.7 MiB 0.0 MiB cycle_length=_num_cores,
192 5766.7 MiB 0.0 MiB num_parallel_calls=tf.data.experimental.AUTOTUNE)
193
194 # dataset = tf.data.TFRecordDataset(files, num_parallel_reads=_num_cores)
195 # dataset = dataset.shuffle(_batch_size*10, 42)
196 # Map the generator output as features as a dict and label
197
198 5766.7 MiB 0.0 MiB dataset = dataset.map(map_func=lambda serialized_example : numpy_array_decode(serialized_example=serialized_example,
199 num_features=num_features),
200 5816.2 MiB 49.5 MiB num_parallel_calls=tf.data.experimental.AUTOTUNE)
201 5816.2 MiB 0.0 MiB dataset = dataset.batch(batch_size=batch_size, drop_remainder=False)
202 5816.2 MiB 0.0 MiB dataset = dataset.repeat()
203
204 # dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
205 # iterator = dataset.make_one_shot_iterator()
206 # batch_feats, batch_label = iterator.get_next()
207 # return batch_feats, batch_label
208 5816.2 MiB 0.0 MiB return dataset
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
173 10184.7 MiB 10184.7 MiB @profile
174 def _get_dataset(data_path,
175 batch_size,
176 num_features):
177 """
178 Reads TFRecords, decode and batches them
179 :return: dataset
180 """
181 10184.7 MiB 0.0 MiB _num_cores = 4
182
183 10184.7 MiB 0.0 MiB path = os.path.join(data_path, "*.tfrecords")
184 10184.7 MiB 0.0 MiB path = path.replace("//", "/")
185 10185.0 MiB 0.3 MiB files = tf.data.Dataset.list_files(path)
186 # files = glob.glob(pathname=path)
187
188 # TF dataset APIs
189 10185.0 MiB 0.0 MiB dataset = files.interleave(
190 10185.0 MiB 0.0 MiB tf.data.TFRecordDataset,
191 10185.0 MiB 0.0 MiB cycle_length=_num_cores,
192 10185.0 MiB 0.0 MiB num_parallel_calls=tf.data.experimental.AUTOTUNE)
193
194 # dataset = tf.data.TFRecordDataset(files, num_parallel_reads=_num_cores)
195 # dataset = dataset.shuffle(_batch_size*10, 42)
196 # Map the generator output as features as a dict and label
197
198 10185.0 MiB 0.0 MiB dataset = dataset.map(map_func=lambda serialized_example : numpy_array_decode(serialized_example=serialized_example,
199 num_features=num_features),
200 10215.2 MiB 30.2 MiB num_parallel_calls=tf.data.experimental.AUTOTUNE)
201 10215.2 MiB 0.0 MiB dataset = dataset.batch(batch_size=batch_size, drop_remainder=False)
202 10215.2 MiB 0.0 MiB dataset = dataset.repeat()
203
204 # dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
205 # iterator = dataset.make_one_shot_iterator()
206 # batch_feats, batch_label = iterator.get_next()
207 # return batch_feats, batch_label
208 10215.2 MiB 0.0 MiB return dataset
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
367 300.7 MiB 300.7 MiB @profile
368 def train_n_evaluate(estimator,
369 train_data_path,
370 val_data_path,
371 batch_size,
372 num_features,
373 num_epochs=None,
374 max_train_steps=None,
375 max_val_steps=None):
376 300.7 MiB 0.0 MiB train_spec = _get_train_spec(train_data_path=train_data_path,
377 300.7 MiB 0.0 MiB batch_size=batch_size,
378 300.7 MiB 0.0 MiB num_features=num_features,
379 300.7 MiB 0.0 MiB num_epochs=num_epochs,
380 420.7 MiB 119.9 MiB max_steps=max_train_steps)
381 420.7 MiB 0.0 MiB eval_spec = _get_eval_spec(val_data_path=val_data_path,
382 420.7 MiB 0.0 MiB batch_size=batch_size,
383 420.7 MiB 0.0 MiB num_features=num_features,
384 420.7 MiB 0.0 MiB num_epochs=num_epochs,
385 420.7 MiB 0.0 MiB max_steps=max_val_steps)
386
387 10256.1 MiB 9835.4 MiB tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
Filename: test_memory_leak.py
Line # Mem usage Increment Line Contents
================================================
407 242.0 MiB 242.0 MiB @profile
408 def main(args):
409 242.0 MiB 0.1 MiB memory_usage_psutil("1. Before generating data")
410
411 # 1. Generate regression data
412 242.0 MiB 0.0 MiB generate_numpy_tf_records(out_dir=args["train_path"],
413 242.0 MiB 0.0 MiB num_tfrecord_files=args["num_tfrecord_train_files"],
414 242.0 MiB 0.0 MiB num_samples_per_file=args["num_samples_per_file"],
415 262.2 MiB 20.2 MiB num_features=args["num_features"])
416 262.2 MiB 0.0 MiB generate_numpy_tf_records(out_dir=args["val_path"],
417 262.2 MiB 0.0 MiB num_tfrecord_files=2,
418 262.2 MiB 0.0 MiB num_samples_per_file=args["num_samples_per_file"],
419 300.5 MiB 38.3 MiB num_features=args["num_features"])
420
421 300.5 MiB 0.0 MiB total_steps_per_file = args["num_samples_per_file"] / args["batch_size"]
INFO:absl:[32m>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>[0m
WARNING:absl:[33mFinal memory usage: : Memory used is 10266.5546875[0m
INFO:absl:[32m<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<[0m
422
423 300.5 MiB 0.0 MiB memory_usage_psutil("2. Before defining model")
424
425 # 2. Define the model
426 300.5 MiB 0.0 MiB model = NNet()
427
428 300.5 MiB 0.0 MiB memory_usage_psutil("3. Before defining estimator")
429
430 # 3. Define engine to train i.e Estimator
431 300.5 MiB 0.0 MiB estimator = tf.estimator.Estimator(model_fn=model,
432 300.5 MiB 0.0 MiB config=_init_tf_config(total_steps_per_file=total_steps_per_file,
433 300.7 MiB 0.2 MiB model_dir=args["model_dir"]),
434 300.7 MiB 0.0 MiB params=None)
435
436 300.7 MiB 0.0 MiB memory_usage_psutil("4. Before training")
437
438 # 4. Train and evaluate the model with generated regression data
439 300.7 MiB 0.0 MiB train_n_evaluate(estimator=estimator,
440 300.7 MiB 0.0 MiB train_data_path=args["train_path"],
441 300.7 MiB 0.0 MiB val_data_path=args["val_path"],
442 300.7 MiB 0.0 MiB batch_size=args["batch_size"],
443 300.7 MiB 0.0 MiB num_features=args["num_features"],
444 300.7 MiB 0.0 MiB num_epochs=args["num_epochs"],
445 300.7 MiB 0.0 MiB max_train_steps=None,
446 10256.1 MiB 9955.3 MiB max_val_steps=None)
447
448 10256.1 MiB 0.0 MiB memory_usage_psutil("5. Before exporitng the model")
449
450 10266.6 MiB 10.5 MiB export_model(estimator=estimator, model_export_path=args["model_export_path"], num_features=args["num_features"])
--- 307.3540062904358 seconds ---