diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..4f91319 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: yanshengjia +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.gitignore b/.gitignore index 7bbc71c..d1ac641 100644 --- a/.gitignore +++ b/.gitignore @@ -1,101 +1,6 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ +*.idea +*.vscode +*.DS_Store +*.pyc +*.o +*.exe \ No newline at end of file diff --git a/README.md b/README.md index a1ecaaa..c61fecd 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,71 @@ # Machine Learning Road -Machine Learning resources and exercises. +Machine Learning Resources, Practice and Research. +## Disclamier +The resources in this repo are only for educational purpose. Do not use resources in this repo for any form of commercial purpose. -## Resources List +If the author of ebook found your intelligence proprietary violated because of contents in this repo, please contact me and I will remove relevant stuff ASAP. -### Courses +*** -* \ Andrew Ng [[Bilibili]](https://www.bilibili.com/video/av9912938/index_2.html#page=1)[[Youtube]](https://www.youtube.com/playlist?list=PLZ9qNFMHZ-A4rycgrgOYma6zxF4BZGGPW) -* \ Hsuan-Tien Lin [[Bilibili]](https://www.bilibili.com/video/av12463015/)[[Youtube]](https://www.youtube.com/playlist?list=PLXVfgk9fNX2I7tB6oIINGBmW50rrmFTqf) -* \ Hsuan-Tien Lin [[Bilibili]](https://www.bilibili.com/video/av12469267/)[[Youtube]](https://www.youtube.com/playlist?list=PLXVfgk9fNX2IQOYPmqjqWsNUFl2kpk1U2) -* \ Andrew Ng [[Netease]](http://open.163.com/special/opencourse/machinelearning.html)[[Youtube]](https://www.youtube.com/playlist?list=PLA89DCFA6ADACE599) -* \ deeplearning.ai [[Netease]](https://163.lu/nPtn42)[[Coursera]](https://www.coursera.org/specializations/deep-learning) +## Courses -### Books +| Course Name | Institution | Lecturer | Link | Category | +| :----------------------------------------------------------- | :------------------------- | :----------------------------------------------------------- | :------------------------------------------------------------ | :------------------------------------- | +| Machine Learning | Coursera | [Andrew Ng](http://www.andrewng.org/) | [[Coursera]](https://www.coursera.org/learn/machine-learning/)[[Bilibili]](https://www.bilibili.com/video/av9912938/index_2.html#page=1)[[Youtube]](https://www.youtube.com/playlist?list=PLZ9qNFMHZ-A4rycgrgOYma6zxF4BZGGPW) | Machine Learning | +| Machine Learning Foundations | National Taiwan University | [Hsuan-Tien Lin](https://www.csie.ntu.edu.tw/~htlin/) | [[Bilibili]](https://www.bilibili.com/video/av12463015/)[[Youtube]](https://www.youtube.com/playlist?list=PLXVfgk9fNX2I7tB6oIINGBmW50rrmFTqf) | Machine Learning | +| Machine Learning Techniques | National Taiwan University | Hsuan-Tien Lin | [[Bilibili]](https://www.bilibili.com/video/av12469267/)[[Youtube]](https://www.youtube.com/playlist?list=PLXVfgk9fNX2IQOYPmqjqWsNUFl2kpk1U2) | Machine Learning | +| Machine Learning | Stanford | Andrew Ng | [[Netease]](http://open.163.com/special/opencourse/machinelearning.html)[[Youtube]](https://www.youtube.com/playlist?list=PLA89DCFA6ADACE599) | Machine Learning | +| Deep Learning | deeplearning.ai | Andrew Ng | [[Netease]](https://163.lu/nPtn42)[[Coursera]](https://www.coursera.org/specializations/deep-learning) | Deep Learning | +| CS231n: Convolutional Neural Networks for Visual Recognition | Stanford | [Fei-Fei Li](http://vision.stanford.edu/feifeili/) | [[Homepage]](http://cs231n.stanford.edu/)[[Youtube]](https://www.youtube.com/playlist?list=PL3FW7Lu3i5JvHM8ljYj-zLfQRF3EO8sYv) | Deep Learning, Computer Vision | +| CS224n: Natural Language Processing with Deep Learning | Stanford | [Christopher Manning](https://nlp.stanford.edu/manning/) | [[Homepage]](http://web.stanford.edu/class/cs224n/)[[Youtube]](https://www.youtube.com/playlist?list=PL3FW7Lu3i5Jsnh1rnUwq_TcylNr7EkRe6) | Deep Learning, NLP | +| Deep Learning for Natural Language Processing | Oxford University | [Phil Blunsom](http://www.cs.ox.ac.uk/people/Phil.Blunsom) | [[Homepage]](http://www.cs.ox.ac.uk/teaching/courses/2016-2017/dl/)[[Slides]](https://github.com/oxford-cs-deepnlp-2017/lectures) | Deep Learning, NLP | +| Applied Deep Learning / Machine Learning and Having It Deep and Structured | National Taiwan University | [Yun-Nung Chen](http://vivianchen.idv.tw/), [Hung-Yi Lee](http://speech.ee.ntu.edu.tw/~tlkagk/) | [[Homepage]](https://www.csie.ntu.edu.tw/~yvchen/f106-adl/index.html)[[Youtube]](https://www.youtube.com/watch?v=hwUca8gsXdM&feature=youtu.be) | Machine Learning, Deep Learning | +| CS 20: TensorFlow for Deep Learning Research | Stanford | Chip Huyen | [[Homepage]](https://web.stanford.edu/class/cs20si/)[[Github]](https://github.com/chiphuyen/stanford-tensorflow-tutorials) | Deep Learning | +| CS 294: Deep Reinforcement Learning | UC Berkeley | Sergey Levine | [[Homepage]](http://rll.berkeley.edu/deeprlcourse/)[[Youtube]](https://www.youtube.com/playlist?list=PLkFD6_40KJIznC9CDbVTjAF2oyt8_VAe3) | Deep Learning, Reinforcement Learning | +| Neural Networks for NLP | CMU | [Graham Neubig](http://phontron.com/) | [[Homepage]](http://www.phontron.com/class/nn4nlp2018/assignments.html) | NLP, Deep Learning | +| Mathematics of Deep Learning | NYU | Joan Bruna | [[Github]](https://github.com/joanbruna/MathsDL-spring18) | Deep Learning | +| Introduction to NLP | Stanford | [Dan Jurafsky](https://web.stanford.edu/~jurafsky/), [Chris Manning](https://nlp.stanford.edu/manning/) | [[Youtube]](https://www.youtube.com/playlist?list=PL6397E4B26D00A269) | NLP | +| Text Mining and Analytics | UIUC | [ChengXiang Zhai](https://zh.coursera.org/instructor/chengxiangzhai) | [[Coursera]](https://zh.coursera.org/learn/text-mining) | NLP | +| Machine Learning Crash Course with TensorFlow APIs | Google | Google | [[Homepage]](https://developers.google.com/machine-learning/crash-course/) | Machine Learning, Tensorflow | +| CS230: Deep Learning | Stanford | [Andrew Ng](http://www.andrewng.org/), [Kian Katanforoosh](https://www.linkedin.com/in/kiankatan/) | [[Homepage]](http://cs230.stanford.edu/) | Deep Learning | +| Intro to Deep Learning with PyTorch | Facebook AI | Facebook AI | [[Udacity]](https://in.udacity.com/course/deep-learning-pytorch--ud188) | Deep Learning, PyTorch | +| Introduction to Deep Learning | UC Berkeley | [Alex Smola](https://alex.smola.org/), [Mu Li](https://github.com/mli) | [[Youtube]](https://www.youtube.com/playlist?list=PLZSO_6-bSqHQHBCoGaObUljoXAyyqhpFW)[[GitHub]](https://github.com/d2l-ai/berkeley-stat-157) | Deep Learning | +| Foundations of Machine Learning | NYU | [Mehryar Mohri](https://cs.nyu.edu/~mohri/) | [[Homepage]](https://cs.nyu.edu/~mohri/ml18/) | Machine Learning | +| DS1003 Machine Learning | NYU | Julia Kempe, David Rosenberg | [[Homepage]](https://davidrosenberg.github.io/ml2019/#home)[[Slides]](https://davidrosenberg.github.io/ml2019/#lectures) [[Youtube]](https://www.youtube.com/watch?v=U6M0m9c9_Js&feature=youtu.be)[[Assignments]](https://davidrosenberg.github.io/ml2019/#assignments) | Machine Learning | +| TensorFlow in Practice | Coursera | Laurence Moroney | [[Coursera]](https://www.coursera.org/specializations/tensorflow-in-practice) | TensorFlow | +| DS-GA 1008 Deep Learning | NYU | [Yann LeCun](http://yann.lecun.com/), [Alfredo Canziani](https://www.linkedin.com/in/alfredocanziani/) | [[Homepage]](https://atcold.github.io/pytorch-Deep-Learning/) [[YouTube]](https://www.youtube.com/playlist?list=PLLHTzKZzVU9eaEyErdV26ikyolxOsz6mq)[[Bilibili]](https://www.bilibili.com/video/BV197411M7gG/?spm_id_from=333.788.videocard.1) | Deep Learning, PyTorch | +| Deep Learning for Human Language Processing | National Taiwan University | [Hung-yi Lee](https://www.youtube.com/channel/UC2ggjtuuWvxrHHHiaDH1dlQ) | [[Homepage]](http://speech.ee.ntu.edu.tw/~tlkagk/courses_DLHLP20.html) [[YouTube]](https://www.youtube.com/watch?v=nER51ZyJaCQ&list=PLJV_el3uVTsO07RpBYFsXg-bN5Lu0nhdG) | Deep Learning, NLP | -* \<机器学习\> 周志华 -* \ Ian Goodfellow, Yoshua Bengio, Aaron Courville -* \ Tom Mitchell -* \ Christopher Bishop -* \ Trevor Hastie, Robert Tibshirani, Jerome Friedman -* \ Ian H. Witten, Eibe Frank -* \ Sturart J. Russell, Peter Norvig -* \ Kevin P. Murphy +## Books +| Book Name | Author | Link | Category | +| :----------------------------------------------------------- | :----------------------------------------------------------- | :----------------------------------------------------------- | :----------------------------- | +| 机器学习 | 周志华 | [[Amazon]](https://www.amazon.cn/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0-%E5%91%A8%E5%BF%97%E5%8D%8E/dp/B01ARKEV1G/ref=sr_1_1?ie=UTF8&qid=1509470386&sr=8-1&keywords=%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0)[[JD]](https://item.jd.com/11867803.html) | Machine Learning | +| Deep Learning | Ian Goodfellow, Yoshua Bengio, Aaron Courville | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Deep%20Learning.pdf)[[中文版]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0.pdf) | Deep Learning | +| Machine Learning | Tom Mitchell | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Machine%20Learning.pdf) | Machine Learning | +| Pattern Recognition and Machine Learning | Christopher Bishop | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Pattern%20Recognition%20and%20Machine%20Learning.pdf)[[中文版]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0%E4%B8%8E%E6%A8%A1%E5%BC%8F%E8%AF%86%E5%88%AB.pdf) | Machine Learning | +| The Elements of Statistical Learning | Trevor Hastie, Robert Tibshirani, Jerome Friedman | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/The%20Elements%20of%20Statistical%20Learning%20(2nd%20Edition).pdf) | Machine Learning | +| Data Mining: Practical Machine Learning Tools and Techniques | Ian H. Witten, Eibe Frank | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Data%20Mining%20-%20Practical%20Machine%20Learning%20Tools%20and%20Techniques%20(2nd%20Edition).pdf) | Data Mining | +| Artificial Intelligence: A Modern Approach | Sturart J. Russell, Peter Norvig | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Artificial%20Intelligence%20-%20A%20Modern%20Approach%20(3rd%20Edition).pdf) | AI | +| Machine Learning: A Probabilistic Perspective | Kevin P. Murphy | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Machine%20Learning%20-%20A%20Probabilistic%20Perspective.pdf) | Machine Learning | +| Natural Language Processing with Python | Stven Bird, Ewan Klein, Edward Loper | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Natural%20Language%20Processing%20with%20Python.pdf)[[Link]](http://www.nltk.org/book_1ed/) | NLP | +| Getting Started with Tensorflow | Giancarlo Zaccone | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Getting%20Started%20with%20Tensorflow.pdf) | Tensorflow | +| Hands-On Machine Learning with Scikit-Learn and TensorFlow | Aurélien Géron | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Hands%20On%20Machine%20Learning%20with%20Scikit%20Learn%20and%20TensorFlow.pdf)[[Github]](https://github.com/ageron/handson-ml) | Machine Learning | +| Deep Learning with Python | François Chollet | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Deep%20Learning%20with%20Python.pdf)[[Github]](https://github.com/fchollet/deep-learning-with-python-notebooks) | Deep Learning | +| Probabilistic Graphical Models: Principles and Techniques | Daphne Koller, Nir Friedman | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Probabilistic%20Graphical%20Models%20-%20Principles%20and%20Techniques.pdf) | Probabilistic Graphical Model | +| Speech and Language Processing | [Dan Jurafsky](http://web.stanford.edu/people/jurafsky/), [James H. Martin](http://www.cs.colorado.edu/~martin/) | [[Homepage]](https://web.stanford.edu/~jurafsky/slp3/)[[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Speech%20and%20Language%20Processing%20(3rd%20Edition).pdf) | NLP | +| Neural Network Methods for Natural Language Processing | Yoav Goldberg | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Neural%20Network%20Methods%20for%20Natural%20Language%20Processing.pdf) | NLP | +| 统计学习方法 | 李航 | [[Amazon]](https://www.amazon.cn/dp/B007TSFMTA/ref=sr_1_1?ie=UTF8&qid=1524798458&sr=8-1&keywords=%E7%BB%9F%E8%AE%A1%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95) | Machine Learning | +| Natural Language Processing | Jacob Eisenstein | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Natural%20Language%20Processing.pdf) | NLP | +| Dive into Deep Learning 动手学深度学习 | Aston Zhang, Mu Li, Zachary C.Lipton, Alexander J.Smola | [[中文版]](https://github.com/yanshengjia/ml-road/blob/master/resources/d2l-zh.pdf) [[PDF]](https://github.com/yanshengjia/ml-road/blob/master/resources/d2l-en.pdf) [[Website]](http://zh.d2l.ai/) [[GitHub]](https://github.com/d2l-ai/d2l-zh) [[Jupyter]](https://zh.d2l.ai/d2l-zh.zip) | Deep Learning | +| Feature Engineering for Machine Learning | Alice Zheng, Amanda Casari | [[PDF]](https://github.com/yanshengjia/machine-learning-road/blob/master/resources/Feature%20Engineering%20for%20Machine%20Learning.pdf)[[译]](https://github.com/apachecn/feature-engineering-for-ml-zh) | Machine Learning, Feature Engineering | +| Machine Learning Yearning | Andrew Ng | [[译]](https://github.com/yanshengjia/ml-road/blob/master/resources/%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0%E8%AE%AD%E7%BB%83%E7%A7%98%E7%B1%8D.pdf)[[在线阅读]](https://accepteddoge.github.io/machine-learning-yearning-cn/docs/home/) | Machine Learning | +| Foundations of Machine Learning | Mehryar Mohri | [[PDF]](https://github.com/yanshengjia/ml-road/blob/master/resources/Foundations%20of%20Machine%20Learning%20(2nd%20Edition).pdf)[[HomePage]](https://cs.nyu.edu/~mohri/mlbook/) | Machine Learning | + +## Papers + +### NLP + +* Tixier A J P. Notes on Deep Learning for NLP[J]. arXiv preprint arXiv:1808.09772, 2018. [[Link]](https://arxiv.org/pdf/1808.09772.pdf) diff --git a/projects/chinese-character-recognition/README.md b/projects/chinese-character-recognition/README.md new file mode 100644 index 0000000..0fc2d37 --- /dev/null +++ b/projects/chinese-character-recognition/README.md @@ -0,0 +1,4 @@ +# Chinese Character Recognition with Batch Norm + +* top 1 accuracy: 0.9249791286257038 +* top k accuracy 0.9747623788455786 \ No newline at end of file diff --git a/projects/chinese-character-recognition/chinese_character_recognition_bn.py b/projects/chinese-character-recognition/chinese_character_recognition_bn.py new file mode 100644 index 0000000..cf25f5e --- /dev/null +++ b/projects/chinese-character-recognition/chinese_character_recognition_bn.py @@ -0,0 +1,322 @@ +# with batch norm + +import os +import random +import tensorflow.contrib.slim as slim +import time +import logging +import numpy as np +import tensorflow as tf +import pickle +from PIL import Image +from tensorflow.python.ops import control_flow_ops + +os.environ['CUDA_VISIBLE_DEVICES']='1' + +logger = logging.getLogger('Training a chinese write char recognition') +logger.setLevel(logging.INFO) +# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) +logger.addHandler(ch) + +# 设置全局变量 +tf.app.flags.DEFINE_boolean('random_flip_up_down', False, "Whether to random flip up down") +tf.app.flags.DEFINE_boolean('random_brightness', True, "whether to adjust brightness") +tf.app.flags.DEFINE_boolean('random_contrast', True, "whether to random constrast") + +tf.app.flags.DEFINE_integer('charset_size', 3755, "Choose the first `charset_size` characters only.") +tf.app.flags.DEFINE_integer('image_size', 64, "Needs to provide same value as in training.") +tf.app.flags.DEFINE_boolean('gray', True, "whether to change the rbg to gray") +tf.app.flags.DEFINE_integer('max_steps', 16002, 'the max training steps ') +tf.app.flags.DEFINE_integer('eval_steps', 100, "the step num to eval") +tf.app.flags.DEFINE_integer('save_steps', 500, "the steps to save") + +tf.app.flags.DEFINE_string('checkpoint_dir', './checkpoint/', 'the checkpoint dir') +tf.app.flags.DEFINE_string('train_data_dir', './data/train/', 'the train dataset dir') +tf.app.flags.DEFINE_string('test_data_dir', './data/test/', 'the test dataset dir') +tf.app.flags.DEFINE_string('log_dir', './log/', 'the logging dir') + +tf.app.flags.DEFINE_boolean('restore', False, 'whether to restore from checkpoint') +tf.app.flags.DEFINE_boolean('epoch', 1, 'Number of epoches') +tf.app.flags.DEFINE_integer('batch_size', 128, 'Validation batch size') +tf.app.flags.DEFINE_string('mode', 'train', 'Running mode. One of {"train", "valid", "test"}') + +gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9) +FLAGS = tf.app.flags.FLAGS + + +class DataIterator: + def __init__(self, data_dir): + # Set FLAGS.charset_size to a small value if available computation power is limited. + truncate_path = data_dir + ('%05d' % FLAGS.charset_size) + print(truncate_path) + self.image_names = [] + for root, sub_folder, file_list in os.walk(data_dir): + if root < truncate_path: + self.image_names += [os.path.join(root, file_path) for file_path in file_list] + random.shuffle(self.image_names) + self.labels = [int(file_name[len(data_dir):].split(os.sep)[0]) for file_name in self.image_names] + + @property + def size(self): + return len(self.labels) + + @staticmethod + def data_augmentation(images): + if FLAGS.random_flip_up_down: + images = tf.image.random_flip_up_down(images) + if FLAGS.random_brightness: + images = tf.image.random_brightness(images, max_delta=0.3) + if FLAGS.random_contrast: + images = tf.image.random_contrast(images, 0.8, 1.2) + return images + + def input_pipeline(self, batch_size, num_epochs=None, aug=False): + images_tensor = tf.convert_to_tensor(self.image_names, dtype=tf.string) + labels_tensor = tf.convert_to_tensor(self.labels, dtype=tf.int64) + input_queue = tf.train.slice_input_producer([images_tensor, labels_tensor], num_epochs=num_epochs) + + labels = input_queue[1] + images_content = tf.read_file(input_queue[0]) + images = tf.image.convert_image_dtype(tf.image.decode_png(images_content, channels=1), tf.float32) + if aug: + images = self.data_augmentation(images) + new_size = tf.constant([FLAGS.image_size, FLAGS.image_size], dtype=tf.int32) + images = tf.image.resize_images(images, new_size) + image_batch, label_batch = tf.train.shuffle_batch([images, labels], batch_size=batch_size, capacity=50000, + min_after_dequeue=10000) + # print 'image_batch', image_batch.get_shape() + return image_batch, label_batch + + +def build_graph(top_k): + keep_prob = tf.placeholder(dtype=tf.float32, shape=[], name='keep_prob') + images = tf.placeholder(dtype=tf.float32, shape=[None, 64, 64, 1], name='image_batch') + labels = tf.placeholder(dtype=tf.int64, shape=[None], name='label_batch') + is_training = tf.placeholder(dtype=tf.bool, shape=[], name='train_flag') + with tf.device('/gpu:1'): + with slim.arg_scope([slim.conv2d, slim.fully_connected], + normalizer_fn=slim.batch_norm, + normalizer_params={'is_training': is_training}): + conv3_1 = slim.conv2d(images, 64, [3, 3], 1, padding='SAME', scope='conv3_1') + max_pool_1 = slim.max_pool2d(conv3_1, [2, 2], [2, 2], padding='SAME', scope='pool1') + conv3_2 = slim.conv2d(max_pool_1, 128, [3, 3], padding='SAME', scope='conv3_2') + max_pool_2 = slim.max_pool2d(conv3_2, [2, 2], [2, 2], padding='SAME', scope='pool2') + conv3_3 = slim.conv2d(max_pool_2, 256, [3, 3], padding='SAME', scope='conv3_3') + max_pool_3 = slim.max_pool2d(conv3_3, [2, 2], [2, 2], padding='SAME', scope='pool3') + conv3_4 = slim.conv2d(max_pool_3, 512, [3, 3], padding='SAME', scope='conv3_4') + conv3_5 = slim.conv2d(conv3_4, 512, [3, 3], padding='SAME', scope='conv3_5') + max_pool_4 = slim.max_pool2d(conv3_5, [2, 2], [2, 2], padding='SAME', scope='pool4') + + flatten = slim.flatten(max_pool_4) + fc1 = slim.fully_connected(slim.dropout(flatten, keep_prob), 1024, + activation_fn=tf.nn.relu, scope='fc1') + logits = slim.fully_connected(slim.dropout(fc1, keep_prob), FLAGS.charset_size, activation_fn=None, + scope='fc2') + loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels)) + accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(logits, 1), labels), tf.float32)) + + update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) + if update_ops: + updates = tf.group(*update_ops) + loss = control_flow_ops.with_dependencies([updates], loss) + + global_step = tf.get_variable("step", [], initializer=tf.constant_initializer(0.0), trainable=False) + optimizer = tf.train.AdamOptimizer(learning_rate=0.1) + train_op = slim.learning.create_train_op(loss, optimizer, global_step=global_step) + probabilities = tf.nn.softmax(logits) + + tf.summary.scalar('loss', loss) + tf.summary.scalar('accuracy', accuracy) + merged_summary_op = tf.summary.merge_all() + predicted_val_top_k, predicted_index_top_k = tf.nn.top_k(probabilities, k=top_k) + accuracy_in_top_k = tf.reduce_mean(tf.cast(tf.nn.in_top_k(probabilities, labels, top_k), tf.float32)) + + return {'images': images, + 'labels': labels, + 'keep_prob': keep_prob, + 'top_k': top_k, + 'global_step': global_step, + 'train_op': train_op, + 'loss': loss, + 'is_training': is_training, + 'accuracy': accuracy, + 'accuracy_top_k': accuracy_in_top_k, + 'merged_summary_op': merged_summary_op, + 'predicted_distribution': probabilities, + 'predicted_index_top_k': predicted_index_top_k, + 'predicted_val_top_k': predicted_val_top_k} + + +def train(): + logger.info('Begin training') + train_feeder = DataIterator(data_dir='./data/train/') + test_feeder = DataIterator(data_dir='./data/test/') + model_name = 'chinese-rec-model' + with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True)) as sess: + train_images, train_labels = train_feeder.input_pipeline(batch_size=FLAGS.batch_size, aug=True) + test_images, test_labels = test_feeder.input_pipeline(batch_size=FLAGS.batch_size) + graph = build_graph(top_k=1) + saver = tf.train.Saver() + sess.run(tf.global_variables_initializer()) + coord = tf.train.Coordinator() + threads = tf.train.start_queue_runners(sess=sess, coord=coord) + + train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph) + test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/val') + start_step = 0 + if FLAGS.restore: + ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) + if ckpt: + saver.restore(sess, ckpt) + print("restore from the checkpoint {0}".format(ckpt)) + start_step += int(ckpt.split('-')[-1]) + + logger.info(':::Training Start:::') + try: + i = 0 + while not coord.should_stop(): + i += 1 + start_time = time.time() + train_images_batch, train_labels_batch = sess.run([train_images, train_labels]) + feed_dict = {graph['images']: train_images_batch, + graph['labels']: train_labels_batch, + graph['keep_prob']: 0.8, + graph['is_training']: True} + _, loss_val, train_summary, step = sess.run( + [graph['train_op'], graph['loss'], graph['merged_summary_op'], graph['global_step']], + feed_dict=feed_dict) + train_writer.add_summary(train_summary, step) + end_time = time.time() + logger.info("the step {0} takes {1} loss {2}".format(step, end_time - start_time, loss_val)) + if step > FLAGS.max_steps: + break + if step % FLAGS.eval_steps == 1: + test_images_batch, test_labels_batch = sess.run([test_images, test_labels]) + feed_dict = {graph['images']: test_images_batch, + graph['labels']: test_labels_batch, + graph['keep_prob']: 1.0, + graph['is_training']: False} + accuracy_test, test_summary = sess.run([graph['accuracy'], graph['merged_summary_op']], + feed_dict=feed_dict) + if step > 300: + test_writer.add_summary(test_summary, step) + logger.info('===============Eval a batch=======================') + logger.info('the step {0} test accuracy: {1}' + .format(step, accuracy_test)) + logger.info('===============Eval a batch=======================') + if step % FLAGS.save_steps == 1: + logger.info('Save the ckpt of {0}'.format(step)) + saver.save(sess, os.path.join(FLAGS.checkpoint_dir, model_name), + global_step=graph['global_step']) + except tf.errors.OutOfRangeError: + logger.info('==================Train Finished================') + saver.save(sess, os.path.join(FLAGS.checkpoint_dir, model_name), global_step=graph['global_step']) + finally: + coord.request_stop() + coord.join(threads) + + +def validation(): + print('Begin validation') + test_feeder = DataIterator(data_dir='./data/test/') + + final_predict_val = [] + final_predict_index = [] + groundtruth = [] + + with tf.Session() as sess: + test_images, test_labels = test_feeder.input_pipeline(batch_size=FLAGS.batch_size, num_epochs=1) + graph = build_graph(top_k=3) + saver = tf.train.Saver() + + sess.run(tf.global_variables_initializer()) + sess.run(tf.local_variables_initializer()) # initialize test_feeder's inside state + + coord = tf.train.Coordinator() + threads = tf.train.start_queue_runners(sess=sess, coord=coord) + + ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) + if ckpt: + saver.restore(sess, ckpt) + print("restore from the checkpoint {0}".format(ckpt)) + + logger.info(':::Start validation:::') + try: + i = 0 + acc_top_1, acc_top_k = 0.0, 0.0 + while not coord.should_stop(): + i += 1 + start_time = time.time() + test_images_batch, test_labels_batch = sess.run([test_images, test_labels]) + feed_dict = {graph['images']: test_images_batch, + graph['labels']: test_labels_batch, + graph['keep_prob']: 1.0, + graph['is_training']: False} + batch_labels, probs, indices, acc_1, acc_k = sess.run([graph['labels'], + graph['predicted_val_top_k'], + graph['predicted_index_top_k'], + graph['accuracy'], + graph['accuracy_top_k']], feed_dict=feed_dict) + final_predict_val += probs.tolist() + final_predict_index += indices.tolist() + groundtruth += batch_labels.tolist() + acc_top_1 += acc_1 + acc_top_k += acc_k + end_time = time.time() + logger.info("the batch {0} takes {1} seconds, accuracy = {2}(top_1) {3}(top_k)" + .format(i, end_time - start_time, acc_1, acc_k)) + + except tf.errors.OutOfRangeError: + logger.info('==================Validation Finished================') + acc_top_1 = acc_top_1 * FLAGS.batch_size / test_feeder.size + acc_top_k = acc_top_k * FLAGS.batch_size / test_feeder.size + logger.info('top 1 accuracy {0} top k accuracy {1}'.format(acc_top_1, acc_top_k)) + finally: + coord.request_stop() + coord.join(threads) + return {'prob': final_predict_val, 'indices': final_predict_index, 'groundtruth': groundtruth} + + +def inference(image): + print('inference') + temp_image = Image.open(image).convert('L') + temp_image = temp_image.resize((FLAGS.image_size, FLAGS.image_size), Image.ANTIALIAS) + temp_image = np.asarray(temp_image) / 255.0 + temp_image = temp_image.reshape([-1, 64, 64, 1]) + with tf.Session() as sess: + logger.info('========start inference============') + # images = tf.placeholder(dtype=tf.float32, shape=[None, 64, 64, 1]) + # Pass a shadow label 0. This label will not affect the computation graph. + graph = build_graph(top_k=3) + saver = tf.train.Saver() + ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_dir) + if ckpt: + saver.restore(sess, ckpt) + predict_val, predict_index = sess.run([graph['predicted_val_top_k'], graph['predicted_index_top_k']], + feed_dict={graph['images']: temp_image, + graph['keep_prob']: 1.0, + graph['is_training']: False}) + return predict_val, predict_index + + +def main(_): + print(FLAGS.mode) + if FLAGS.mode == "train": + train() + elif FLAGS.mode == 'validation': + dct = validation() + result_file = 'result.dict' + logger.info('Write result into {0}'.format(result_file)) + with open(result_file, 'wb') as f: + pickle.dump(dct, f) + logger.info('Write file ends') + elif FLAGS.mode == 'inference': + image_path = './data/test/00190/13320.png' + final_predict_val, final_predict_index = inference(image_path) + logger.info('the result info label {0} predict index {1} predict_val {2}'.format(190, final_predict_index, + final_predict_val)) + +if __name__ == "__main__": + tf.app.run() \ No newline at end of file diff --git a/projects/mnist-handwritten-digit-recognition/README.md b/projects/mnist-handwritten-digit-recognition/README.md new file mode 100644 index 0000000..4e19f74 --- /dev/null +++ b/projects/mnist-handwritten-digit-recognition/README.md @@ -0,0 +1,7 @@ +# MNIST Handwritten Digit Recognition + +Some experiments on MNIST handwritten digit recognition with Tensorflow. + +## References + +* [MNIST For ML Beginners](https://www.tensorflow.org/get_started/mnist/beginners?hl=zh-cn) \ No newline at end of file diff --git a/projects/mnist-handwritten-digit-recognition/mnist_softmax.py b/projects/mnist-handwritten-digit-recognition/mnist_softmax.py new file mode 100644 index 0000000..e8609ed --- /dev/null +++ b/projects/mnist-handwritten-digit-recognition/mnist_softmax.py @@ -0,0 +1,65 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# @author: Shengjia Yan +# @date: 2018-01-09 Tuesday +# @email: i@yanshengjia.com +# Copyright 2018 Shengjia Yan. All Rights Reserved. + +"""A very simple MNIST classifier. + +See extensive documentation at +https://www.tensorflow.org/get_started/mnist/beginners +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import argparse +import sys +from tensorflow.examples.tutorials.mnist import input_data +import tensorflow as tf + +FLAGS = None + + +def main(_): + # Import data + mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) + + # Create the model + x = tf.placeholder(tf.float32, [None, 784]) + W = tf.Variable(tf.zeros([784, 10])) + b = tf.Variable(tf.zeros([10])) + y = tf.matmul(x, W) + b + + # Define loss and optimizer + y_ = tf.placeholder(tf.float32, [None, 10]) + + # The raw formulation of cross-entropy, + # + # tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), reduction_indices=[1])) + # + # can be numerically unstable. + # + # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw + # outputs of 'y', and then average across the batch. + cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) + train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy) + + sess = tf.InteractiveSession() + tf.global_variables_initializer().run() + + # Train + for _ in range(1000): + batch_xs, batch_ys = mnist.train.next_batch(100) + sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) + + # Test trained model + correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) + accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) + print(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', type=str, default='/tmp/tensorflow/mnist/input_data', help='Directory for storing input data') + FLAGS, unparsed = parser.parse_known_args() + tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) \ No newline at end of file diff --git a/resources/Deep Learning with Python.pdf b/resources/Deep Learning with Python.pdf new file mode 100644 index 0000000..f3c7aa6 Binary files /dev/null and b/resources/Deep Learning with Python.pdf differ diff --git a/resources/Feature Engineering for Machine Learning.pdf b/resources/Feature Engineering for Machine Learning.pdf new file mode 100644 index 0000000..313045c Binary files /dev/null and b/resources/Feature Engineering for Machine Learning.pdf differ diff --git a/resources/Foundations of Machine Learning (2nd Edition).pdf b/resources/Foundations of Machine Learning (2nd Edition).pdf new file mode 100644 index 0000000..6956124 Binary files /dev/null and b/resources/Foundations of Machine Learning (2nd Edition).pdf differ diff --git a/resources/Getting Started with Tensorflow.pdf b/resources/Getting Started with Tensorflow.pdf new file mode 100644 index 0000000..fc3ae15 Binary files /dev/null and b/resources/Getting Started with Tensorflow.pdf differ diff --git a/resources/Hands On Machine Learning with Scikit Learn and TensorFlow.pdf b/resources/Hands On Machine Learning with Scikit Learn and TensorFlow.pdf new file mode 100644 index 0000000..8b9fc27 Binary files /dev/null and b/resources/Hands On Machine Learning with Scikit Learn and TensorFlow.pdf differ diff --git a/resources/ISLR Seventh Printing.pdf b/resources/ISLR Seventh Printing.pdf new file mode 100644 index 0000000..4796dcc Binary files /dev/null and b/resources/ISLR Seventh Printing.pdf differ diff --git a/resources/Natural Language Processing with Python.pdf b/resources/Natural Language Processing with Python.pdf new file mode 100644 index 0000000..038a3cb Binary files /dev/null and b/resources/Natural Language Processing with Python.pdf differ diff --git a/resources/Natural Language Processing.pdf b/resources/Natural Language Processing.pdf new file mode 100644 index 0000000..034450d Binary files /dev/null and b/resources/Natural Language Processing.pdf differ diff --git a/resources/Neural Network Methods for Natural Language Processing.pdf b/resources/Neural Network Methods for Natural Language Processing.pdf new file mode 100644 index 0000000..31f5149 Binary files /dev/null and b/resources/Neural Network Methods for Natural Language Processing.pdf differ diff --git a/resources/Probabilistic Graphical Models - Principles and Techniques.pdf b/resources/Probabilistic Graphical Models - Principles and Techniques.pdf new file mode 100644 index 0000000..2b77104 Binary files /dev/null and b/resources/Probabilistic Graphical Models - Principles and Techniques.pdf differ diff --git a/resources/Speech and Language Processing (3rd Edition).pdf b/resources/Speech and Language Processing (3rd Edition).pdf new file mode 100644 index 0000000..0c04a48 Binary files /dev/null and b/resources/Speech and Language Processing (3rd Edition).pdf differ diff --git a/resources/d2l-en.pdf b/resources/d2l-en.pdf new file mode 100644 index 0000000..3f4be80 Binary files /dev/null and b/resources/d2l-en.pdf differ diff --git a/resources/d2l-zh.pdf b/resources/d2l-zh.pdf new file mode 100644 index 0000000..d0ee3af Binary files /dev/null and b/resources/d2l-zh.pdf differ diff --git a/resources/gluon_tutorials_zh.pdf b/resources/gluon_tutorials_zh.pdf new file mode 100644 index 0000000..1f986ff Binary files /dev/null and b/resources/gluon_tutorials_zh.pdf differ diff --git "a/resources/\346\234\272\345\231\250\345\255\246\344\271\240\350\256\255\347\273\203\347\247\230\347\261\215.pdf" "b/resources/\346\234\272\345\231\250\345\255\246\344\271\240\350\256\255\347\273\203\347\247\230\347\261\215.pdf" new file mode 100644 index 0000000..490cd9c Binary files /dev/null and "b/resources/\346\234\272\345\231\250\345\255\246\344\271\240\350\256\255\347\273\203\347\247\230\347\261\215.pdf" differ diff --git "a/resources/\346\234\272\345\231\250\345\255\246\344\271\240\344\270\216\346\250\241\345\274\217\350\257\206\345\210\253.pdf" "b/resources/\346\250\241\345\274\217\350\257\206\345\210\253\344\270\216\346\234\272\345\231\250\345\255\246\344\271\240.pdf" similarity index 100% rename from "resources/\346\234\272\345\231\250\345\255\246\344\271\240\344\270\216\346\250\241\345\274\217\350\257\206\345\210\253.pdf" rename to "resources/\346\250\241\345\274\217\350\257\206\345\210\253\344\270\216\346\234\272\345\231\250\345\255\246\344\271\240.pdf" diff --git "a/resources/\346\267\261\345\272\246\345\255\246\344\271\240.pdf" "b/resources/\346\267\261\345\272\246\345\255\246\344\271\240.pdf" new file mode 100644 index 0000000..fd472d6 Binary files /dev/null and "b/resources/\346\267\261\345\272\246\345\255\246\344\271\240.pdf" differ