20_Bert_intro.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "362da637-4b36-4920-bb02-15a87db740fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mERROR: Could not find a version that satisfies the requirement tensorflow-text (from versions: none)\u001b[0m\u001b[31m\n",
      "\u001b[0m\u001b[31mERROR: No matching distribution found for tensorflow-text\u001b[0m\u001b[31m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "#!pip install tensorflow-text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d83dd248-8394-47bf-8abb-fc3a2c59eee5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow_hub as hub\n",
    "#import tensorflow_text as text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0c10d6d0-2a6c-4afc-a0d2-378f89b90932",
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocess_url = \"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\"\n",
    "encoder_url = \"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0173b5de-b0cd-42bb-b23e-1541f5a6148b",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metal device set to: Apple M1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-02-18 16:42:23.929968: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2023-02-18 16:42:23.930070: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "Op type not registered 'CaseFoldUTF8' in binary running on 192.168.1.111. Make sure the Op and Kernel are registered in the binary running in this process. Note that if you are loading a saved graph which used ops from tf.contrib, accessing (e.g.) `tf.contrib.resampler` should be done before importing the graph, as contrib ops are lazily registered when the module is first accessed.\n You may be trying to load on a different device from the computational device. Consider setting the `experimental_io_device` option in `tf.saved_model.LoadOptions` to the io_device such as '/job:localhost'.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow/python/framework/ops.py:4155\u001b[0m, in \u001b[0;36mGraph._get_op_def\u001b[0;34m(self, type)\u001b[0m\n\u001b[1;32m   4154\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 4155\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_op_def_cache\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m   4156\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m:\n",
      "\u001b[0;31mKeyError\u001b[0m: 'CaseFoldUTF8'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mNotFoundError\u001b[0m                             Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow/python/saved_model/load.py:912\u001b[0m, in \u001b[0;36mload_partial\u001b[0;34m(export_dir, filters, tags, options)\u001b[0m\n\u001b[1;32m    911\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 912\u001b[0m   loader \u001b[38;5;241m=\u001b[39m \u001b[43mLoader\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobject_graph_proto\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msaved_model_proto\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mexport_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    913\u001b[0m \u001b[43m                  \u001b[49m\u001b[43mckpt_options\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfilters\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    914\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m errors\u001b[38;5;241m.\u001b[39mNotFoundError \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow/python/saved_model/load.py:151\u001b[0m, in \u001b[0;36mLoader.__init__\u001b[0;34m(self, object_graph_proto, saved_model_proto, export_dir, ckpt_options, save_options, filters)\u001b[0m\n\u001b[1;32m    149\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_export_dir \u001b[38;5;241m=\u001b[39m export_dir\n\u001b[1;32m    150\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_concrete_functions \u001b[38;5;241m=\u001b[39m (\n\u001b[0;32m--> 151\u001b[0m     \u001b[43mfunction_deserialization\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_function_def_library\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    152\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlibrary\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmeta_graph\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgraph_def\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlibrary\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    153\u001b[0m \u001b[43m        \u001b[49m\u001b[43msaved_object_graph\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_proto\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    154\u001b[0m \u001b[43m        \u001b[49m\u001b[43mwrapper_function\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_WrapperFunction\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m    155\u001b[0m \u001b[38;5;66;03m# Store a set of all concrete functions that have been set up with\u001b[39;00m\n\u001b[1;32m    156\u001b[0m \u001b[38;5;66;03m# captures.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow/python/saved_model/function_deserialization.py:409\u001b[0m, in \u001b[0;36mload_function_def_library\u001b[0;34m(library, saved_object_graph, load_shared_name_suffix, wrapper_function)\u001b[0m\n\u001b[1;32m    408\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m graph\u001b[38;5;241m.\u001b[39mas_default():\n\u001b[0;32m--> 409\u001b[0m   func_graph \u001b[38;5;241m=\u001b[39m \u001b[43mfunction_def_lib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfunction_def_to_graph\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    410\u001b[0m \u001b[43m      \u001b[49m\u001b[43mfdef\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    411\u001b[0m \u001b[43m      \u001b[49m\u001b[43mstructured_input_signature\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstructured_input_signature\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    412\u001b[0m \u001b[43m      \u001b[49m\u001b[43mstructured_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstructured_outputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    413\u001b[0m \u001b[38;5;66;03m# Restores gradients for function-call ops (not the same as ops that use\u001b[39;00m\n\u001b[1;32m    414\u001b[0m \u001b[38;5;66;03m# custom gradients)\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow/python/framework/function_def_to_graph.py:82\u001b[0m, in \u001b[0;36mfunction_def_to_graph\u001b[0;34m(fdef, structured_input_signature, structured_outputs, input_shapes)\u001b[0m\n\u001b[1;32m     80\u001b[0m         input_shapes\u001b[38;5;241m.\u001b[39mappend(input_shape)\n\u001b[0;32m---> 82\u001b[0m graph_def, nested_to_flat_tensor_name \u001b[38;5;241m=\u001b[39m \u001b[43mfunction_def_to_graph_def\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     83\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfdef\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minput_shapes\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     85\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m func_graph\u001b[38;5;241m.\u001b[39mas_default():\n\u001b[1;32m     86\u001b[0m   \u001b[38;5;66;03m# Add all function nodes to the graph.\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow/python/framework/function_def_to_graph.py:252\u001b[0m, in \u001b[0;36mfunction_def_to_graph_def\u001b[0;34m(fdef, input_shapes)\u001b[0m\n\u001b[1;32m    251\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 252\u001b[0m   op_def \u001b[38;5;241m=\u001b[39m \u001b[43mdefault_graph\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_op_def\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnode_def\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mop\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# pylint: disable=protected-access\u001b[39;00m\n\u001b[1;32m    254\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m attr \u001b[38;5;129;01min\u001b[39;00m op_def\u001b[38;5;241m.\u001b[39mattr:\n",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow/python/framework/ops.py:4159\u001b[0m, in \u001b[0;36mGraph._get_op_def\u001b[0;34m(self, type)\u001b[0m\n\u001b[1;32m   4157\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m c_api_util\u001b[38;5;241m.\u001b[39mtf_buffer() \u001b[38;5;28;01mas\u001b[39;00m buf:\n\u001b[1;32m   4158\u001b[0m   \u001b[38;5;66;03m# pylint: disable=protected-access\u001b[39;00m\n\u001b[0;32m-> 4159\u001b[0m   \u001b[43mpywrap_tf_session\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTF_GraphGetOpDef\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_c_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcompat\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_bytes\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mtype\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   4160\u001b[0m \u001b[43m                                     \u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   4161\u001b[0m   \u001b[38;5;66;03m# pylint: enable=protected-access\u001b[39;00m\n",
      "\u001b[0;31mNotFoundError\u001b[0m: Op type not registered 'CaseFoldUTF8' in binary running on 192.168.1.111. Make sure the Op and Kernel are registered in the binary running in this process. Note that if you are loading a saved graph which used ops from tf.contrib, accessing (e.g.) `tf.contrib.resampler` should be done before importing the graph, as contrib ops are lazily registered when the module is first accessed.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m bert_preprocess_model \u001b[38;5;241m=\u001b[39m \u001b[43mhub\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mKerasLayer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpreprocess_url\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow_hub/keras_layer.py:153\u001b[0m, in \u001b[0;36mKerasLayer.__init__\u001b[0;34m(self, handle, trainable, arguments, _sentinel, tags, signature, signature_outputs_as_dict, output_key, output_shape, load_options, **kwargs)\u001b[0m\n\u001b[1;32m    149\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_output_shape \u001b[38;5;241m=\u001b[39m data_structures\u001b[38;5;241m.\u001b[39mNoDependency(\n\u001b[1;32m    150\u001b[0m       _convert_nest_to_shapes(output_shape))\n\u001b[1;32m    152\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_load_options \u001b[38;5;241m=\u001b[39m load_options\n\u001b[0;32m--> 153\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_func \u001b[38;5;241m=\u001b[39m \u001b[43mload_module\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_load_options\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    154\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_has_training_argument \u001b[38;5;241m=\u001b[39m func_has_training_argument(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_func)\n\u001b[1;32m    155\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_is_hub_module_v1 \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_func, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_is_hub_module_v1\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mFalse\u001b[39;00m)\n",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow_hub/keras_layer.py:449\u001b[0m, in \u001b[0;36mload_module\u001b[0;34m(handle, tags, load_options)\u001b[0m\n\u001b[1;32m    447\u001b[0m   \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:  \u001b[38;5;66;03m# Expected before TF2.4.\u001b[39;00m\n\u001b[1;32m    448\u001b[0m     set_load_options \u001b[38;5;241m=\u001b[39m load_options\n\u001b[0;32m--> 449\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmodule_v2\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhandle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtags\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mset_load_options\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow_hub/module_v2.py:106\u001b[0m, in \u001b[0;36mload\u001b[0;34m(handle, tags, options)\u001b[0m\n\u001b[1;32m    103\u001b[0m   obj \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mcompat\u001b[38;5;241m.\u001b[39mv1\u001b[38;5;241m.\u001b[39msaved_model\u001b[38;5;241m.\u001b[39mload_v2(\n\u001b[1;32m    104\u001b[0m       module_path, tags\u001b[38;5;241m=\u001b[39mtags, options\u001b[38;5;241m=\u001b[39moptions)\n\u001b[1;32m    105\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 106\u001b[0m   obj \u001b[38;5;241m=\u001b[39m \u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompat\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mv1\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msaved_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_v2\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmodule_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtags\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtags\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    107\u001b[0m obj\u001b[38;5;241m.\u001b[39m_is_hub_module_v1 \u001b[38;5;241m=\u001b[39m is_hub_module_v1  \u001b[38;5;66;03m# pylint: disable=protected-access\u001b[39;00m\n\u001b[1;32m    108\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj\n",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow/python/saved_model/load.py:782\u001b[0m, in \u001b[0;36mload\u001b[0;34m(export_dir, tags, options)\u001b[0m\n\u001b[1;32m    780\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(export_dir, os\u001b[38;5;241m.\u001b[39mPathLike):\n\u001b[1;32m    781\u001b[0m   export_dir \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mfspath(export_dir)\n\u001b[0;32m--> 782\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mload_partial\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexport_dir\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mroot\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m    783\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/miniconda3/envs/tfm1_arm64/lib/python3.9/site-packages/tensorflow/python/saved_model/load.py:915\u001b[0m, in \u001b[0;36mload_partial\u001b[0;34m(export_dir, filters, tags, options)\u001b[0m\n\u001b[1;32m    912\u001b[0m   loader \u001b[38;5;241m=\u001b[39m Loader(object_graph_proto, saved_model_proto, export_dir,\n\u001b[1;32m    913\u001b[0m                   ckpt_options, options, filters)\n\u001b[1;32m    914\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m errors\u001b[38;5;241m.\u001b[39mNotFoundError \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 915\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mFileNotFoundError\u001b[39;00m(\n\u001b[1;32m    916\u001b[0m       \u001b[38;5;28mstr\u001b[39m(err) \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m You may be trying to load on a different device \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    917\u001b[0m       \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfrom the computational device. Consider setting the \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    918\u001b[0m       \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`experimental_io_device` option in `tf.saved_model.LoadOptions` \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    919\u001b[0m       \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mto the io_device such as \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/job:localhost\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    920\u001b[0m root \u001b[38;5;241m=\u001b[39m loader\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;241m0\u001b[39m)\n\u001b[1;32m    921\u001b[0m root\u001b[38;5;241m.\u001b[39mgraph_debug_info \u001b[38;5;241m=\u001b[39m loader\u001b[38;5;241m.\u001b[39madjust_debug_info_func_names(debug_info)\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: Op type not registered 'CaseFoldUTF8' in binary running on 192.168.1.111. Make sure the Op and Kernel are registered in the binary running in this process. Note that if you are loading a saved graph which used ops from tf.contrib, accessing (e.g.) `tf.contrib.resampler` should be done before importing the graph, as contrib ops are lazily registered when the module is first accessed.\n You may be trying to load on a different device from the computational device. Consider setting the `experimental_io_device` option in `tf.saved_model.LoadOptions` to the io_device such as '/job:localhost'."
     ]
    }
   ],
   "source": [
    "bert_preprocess_model = hub.KerasLayer(preprocess_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da901093-33b6-4b69-8ca4-b12c4d7be1a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_test = ['nice movie indeed','I love python programming']\n",
    "text_preprocessed = bert_preprocess_model(text_test)\n",
    "text_preprocessed.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4a722c9-6975-478a-abc4-015a3dad94c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_preprocessed['input_mask']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d815dc18-b507-4123-a9d4-2fd9e4f96ca0",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_preprocessed['input_type_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b62b0608-734a-4e4c-9b54-98831cf65100",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_preprocessed['input_word_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e4a7c0a-fd8c-4f46-9311-8c83d982c21c",
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_model = hub.KerasLayer(encoder_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d57f06e-3a64-49b6-a7d1-c7b8fd8ac128",
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_results = bert_model(text_preprocessed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb1f7141-1c69-48b2-af3a-5dc3262ff4a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_results.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a0c6dde-aa86-411b-9e91-036231e668dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_results['sequence_output']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7f7dd5ea-a09f-4b29-b886-ad384ec6d342",
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_results['pooled_output']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcca4294-144e-4027-a30f-b56386333bc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(bert_results['encoder_outputs'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd2dc942-ca6f-4168-a43e-648bf8c405f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_results['encoder_outputs'][-1] == bert_results['sequence_output']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77705675-447c-42f1-bb4d-14ae413db160",
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_results['encoder_outputs']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}