examples/tutorial/BERTRegressor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "recent-scene",
   "metadata": {},
   "source": [
    "# BERTRegressor\n",
    "\n",
    "可用的中文预训练参数：[`bert-base`](https://storage.googleapis.com/bert_models/2018_11_03/chinese_L-12_H-768_A-12.zip),[`roberta-wwm-ext-base`](https://drive.google.com/uc?export=download&id=1jMAKIJmPn7kADgD3yQZhpsqM-IRM1qZt),[`roberta-wwm-ext-large`](https://drive.google.com/uc?export=download&id=1dtad0FFzG11CBsawu8hvwwzU2R0FDI94),[`macbert-base`](https://drive.google.com/uc?export=download&id=1aV69OhYzIwj_hn-kO1RiBa-m8QAusQ5b),[`macbert-large`](https://drive.google.com/uc?export=download&id=1lWYxnk1EqTA2Q20_IShxBrCPc5VSDCkT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "international-highlight",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "v2.5.16\n"
     ]
    }
   ],
   "source": [
    "import uf\n",
    "\n",
    "print(uf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "behavioral-commander",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "uf.BERTRegressor(\n",
      "    config_file=\"../../ref/bert_config.json\",\n",
      "    vocab_file=\"../../ref/vocab.txt\",\n",
      "    max_seq_length=128,\n",
      "    label_size=None,\n",
      "    init_checkpoint=None,\n",
      "    output_dir=None,\n",
      "    gpu_ids=None,\n",
      "    do_lower_case=True,\n",
      "    truncate_method=\"LIFO\",\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model = uf.BERTRegressor(\"../../ref/bert_config.json\", \"../../ref/vocab.txt\")\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "affiliated-medication",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [\"天亮以前说再见\", \"笑着泪流满面\"]\n",
    "y = [[-0.2, 0.3], [0.5, -0.1]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "interracial-charleston",
   "metadata": {},
   "source": [
    "# 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "stainless-bouquet",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.16-py3.8.egg/uf/apps/bert/bert.py:235: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  query_layer = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.16-py3.8.egg/uf/apps/bert/bert.py:245: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  key_layer = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.16-py3.8.egg/uf/apps/bert/bert.py:255: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  value_layer = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.16-py3.8.egg/uf/apps/bert/bert.py:379: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  attention_output = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.16-py3.8.egg/uf/apps/bert/bert.py:391: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  intermediate_output = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.16-py3.8.egg/uf/apps/bert/bert.py:401: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  layer_output = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.16-py3.8.egg/uf/apps/bert/bert.py:102: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  self.pooled_output = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.16-py3.8.egg/uf/apps/_base_/_base_regressor.py:32: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  intermediate_output = tf.layers.dense(\n",
      "/Users/geyingli/Library/Python/3.8/lib/python/site-packages/uf-2.5.16-py3.8.egg/uf/apps/_base_/_base_regressor.py:39: UserWarning: `tf.layers.dense` is deprecated and will be removed in a future version. Please use `tf.keras.layers.Dense` instead.\n",
      "  logits = tf.layers.dense(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Build graph with 306,821,424 parameters (among which 102,273,808 are trainable)\n",
      "INFO:tensorflow:Running local_init_op\n",
      "INFO:tensorflow:Done running local_init_op\n",
      "INFO:tensorflow:Running training on 2 samples (step 0 -> 20)\n",
      "INFO:tensorflow:step 1, mse 0.099286, 0.12 steps/sec, 0.24 examples/sec\n",
      "INFO:tensorflow:step 2, mse 0.096180, 0.34 steps/sec, 0.68 examples/sec\n",
      "INFO:tensorflow:step 3, mse 0.084042, 0.59 steps/sec, 1.17 examples/sec\n",
      "INFO:tensorflow:step 4, mse 0.078840, 0.59 steps/sec, 1.19 examples/sec\n",
      "INFO:tensorflow:step 5, mse 0.077744, 0.55 steps/sec, 1.10 examples/sec\n",
      "INFO:tensorflow:step 6, mse 0.073899, 0.56 steps/sec, 1.13 examples/sec\n",
      "INFO:tensorflow:step 7, mse 0.072024, 0.57 steps/sec, 1.14 examples/sec\n",
      "INFO:tensorflow:step 8, mse 0.067844, 0.55 steps/sec, 1.10 examples/sec\n",
      "INFO:tensorflow:step 9, mse 0.064852, 0.53 steps/sec, 1.05 examples/sec\n",
      "INFO:tensorflow:step 10, mse 0.058260, 0.55 steps/sec, 1.11 examples/sec\n",
      "INFO:tensorflow:step 11, mse 0.052510, 0.59 steps/sec, 1.18 examples/sec\n",
      "INFO:tensorflow:step 12, mse 0.040923, 0.59 steps/sec, 1.18 examples/sec\n",
      "INFO:tensorflow:step 13, mse 0.032865, 0.58 steps/sec, 1.16 examples/sec\n",
      "INFO:tensorflow:step 14, mse 0.025718, 0.56 steps/sec, 1.12 examples/sec\n",
      "INFO:tensorflow:step 15, mse 0.023563, 0.57 steps/sec, 1.15 examples/sec\n",
      "INFO:tensorflow:step 16, mse 0.019170, 0.59 steps/sec, 1.17 examples/sec\n",
      "INFO:tensorflow:step 17, mse 0.019022, 0.58 steps/sec, 1.17 examples/sec\n",
      "INFO:tensorflow:step 18, mse 0.019608, 0.59 steps/sec, 1.17 examples/sec\n",
      "INFO:tensorflow:step 19, mse 0.016995, 0.58 steps/sec, 1.16 examples/sec\n",
      "INFO:tensorflow:step 20, mse 0.016637, 0.58 steps/sec, 1.16 examples/sec\n"
     ]
    }
   ],
   "source": [
    "model.fit(X, y, total_steps=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "primary-enlargement",
   "metadata": {},
   "source": [
    "# 推理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "joined-virgin",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Running inference on 2 samples\n",
      "INFO:tensorflow:process 100.0%, 1.43 examples/sec\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'probs': array([[-0.16144554,  0.18118373],\n",
       "        [ 0.28492033, -0.07230819]], dtype=float32)}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(X)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.2 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "vscode": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}