agi-computer-control-lstm-unified-encoding.ipynb

{"cells":[{"cell_type":"markdown","metadata":{},"source":["you can save a version of the notebook, then copy and edit the version.\n","\n","# Useful Library:\n","\n","[timm](https://github.com/huggingface/pytorch-image-models): pytorch image models"]},{"cell_type":"markdown","metadata":{},"source":["you'd use einops to flatten the tensor.\n","since you would doing operation separately.\n","\n","to measure the overall loss, you would consider merging two types of tensor into one."]},{"cell_type":"markdown","metadata":{},"source":["# Code"]},{"cell_type":"markdown","metadata":{},"source":["## import some library and setup data filepaths"]},{"cell_type":"code","execution_count":155,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:25.407358Z","iopub.status.busy":"2023-04-22T15:36:25.406215Z","iopub.status.idle":"2023-04-22T15:36:37.562085Z","shell.execute_reply":"2023-04-22T15:36:37.559965Z","shell.execute_reply.started":"2023-04-22T15:36:25.407309Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Requirement already satisfied: einops in /opt/conda/lib/python3.7/site-packages (0.6.1)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0m"]}],"source":["!pip3 install einops\n","import einops"]},{"cell_type":"code","execution_count":156,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:37.565926Z","iopub.status.busy":"2023-04-22T15:36:37.565444Z","iopub.status.idle":"2023-04-22T15:36:37.574885Z","shell.execute_reply":"2023-04-22T15:36:37.573375Z","shell.execute_reply.started":"2023-04-22T15:36:37.565878Z"},"trusted":true},"outputs":[],"source":["# try to train a model.\n","# first let's get the data.\n","\n","# if you want to use the template generated by gpt4, you have to use kaggle.\n","\n","##############################\n","## UPLOAD TO KAGGLE DATASET ##\n","##############################\n","import os\n","\n","basePath = \"/kaggle/input/agi-computer-control-test-dataset\"\n","videoPath = os.path.join(basePath, \"93755268.mp4\")\n","# videoPath = \"ffmpeg_output_test_hwaccel.mp4\"\n","metadata_file_path = os.path.join(basePath,\"screenshot_and_actions.json\")\n","\n","# keep it simple, just want to scratch the surface.\n","# we can run this on cpu.\n","# from PIL import Image\n","import numpy as np\n","import cv2\n","import ast\n","from pydantic import BaseModel, validator\n","from typing import Union\n","try:\n","    from typing import Literal\n","except:\n","    from typing_extensions import Literal # this is a failsafe."]},{"cell_type":"markdown","metadata":{},"source":["## define some data models"]},{"cell_type":"code","execution_count":157,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:37.578806Z","iopub.status.busy":"2023-04-22T15:36:37.578272Z","iopub.status.idle":"2023-04-22T15:36:37.688255Z","shell.execute_reply":"2023-04-22T15:36:37.686625Z","shell.execute_reply.started":"2023-04-22T15:36:37.578762Z"},"trusted":true},"outputs":[],"source":["# import pynput\n","# no such dependency when training.\n","\n","class HIDActionBase:\n","    mouse_resolution: int = 1000\n","    keyboard_action_types = [\n","        \"key_press\",\n","        \"key_release\",\n","    ]\n","    mouse_action_types = [\n","        \"mouse_move\",\n","        \"mouse_click\",\n","        \"mouse_scroll\",\n","    ]\n","    action_types = [\n","        *keyboard_action_types,\n","        *mouse_action_types,\n","        # None,  # end of action\n","        # there is no such thing here. do it externally.\n","    ]\n","    mouse_buttons = [\n","        \"Button.left\",\n","        \"Button.middle\",\n","        \"Button.right\",\n","    ]\n","    keys = [\n","        \"\"\"','\"\"\",\n","        \"\"\"'.'\"\"\",\n","        \"\"\"'/'\"\"\",\n","        \"\"\"';'\"\"\",\n","        \"\"\"\\\"'\\\"\"\"\",\n","        \"\"\"'['\"\"\",\n","        \"\"\"']'\"\"\",\n","        \"\"\"'\\\\'\"\"\",\n","        \"\"\"'='\"\"\",\n","        \"\"\"'-'\"\"\",\n","        \"\"\"'0'\"\"\",\n","        \"\"\"'9'\"\"\",\n","        \"\"\"'8'\"\"\",\n","        \"\"\"'7'\"\"\",\n","        \"\"\"'6'\"\"\",\n","        \"\"\"'5'\"\"\",\n","        \"\"\"'4'\"\"\",\n","        \"\"\"'3'\"\"\",\n","        \"\"\"'2'\"\"\",\n","        \"\"\"'1'\"\"\",\n","        \"\"\"'`'\"\"\",\n","        \"\"\"'a'\"\"\",\n","        \"\"\"'b'\"\"\",\n","        \"\"\"'c'\"\"\",\n","        \"\"\"'d'\"\"\",\n","        \"\"\"'e'\"\"\",\n","        \"\"\"'f'\"\"\",\n","        \"\"\"'g'\"\"\",\n","        \"\"\"'h'\"\"\",\n","        \"\"\"'i'\"\"\",\n","        \"\"\"'j'\"\"\",\n","        \"\"\"'k'\"\"\",\n","        \"\"\"'l'\"\"\",\n","        \"\"\"'m'\"\"\",\n","        \"\"\"'n'\"\"\",\n","        \"\"\"'o'\"\"\",\n","        \"\"\"'p'\"\"\",\n","        \"\"\"'q'\"\"\",\n","        \"\"\"'r'\"\"\",\n","        \"\"\"'s'\"\"\",\n","        \"\"\"'t'\"\"\",\n","        \"\"\"'u'\"\"\",\n","        \"\"\"'v'\"\"\",\n","        \"\"\"'w'\"\"\",\n","        \"\"\"'x'\"\"\",\n","        \"\"\"'y'\"\"\",\n","        \"\"\"'z'\"\"\",\n","        \"Key.alt\",\n","        \"Key.alt\",\n","        \"Key.alt_r\",\n","        \"Key.alt_r\",\n","        \"Key.backspace\",\n","        \"Key.caps_lock\",\n","        \"Key.cmd\",\n","        \"Key.cmd\",\n","        \"Key.cmd_r\",\n","        \"Key.ctrl\",\n","        \"Key.ctrl\",\n","        \"Key.ctrl_r\",\n","        \"Key.delete\",\n","        \"Key.down\",\n","        \"Key.end\",\n","        \"Key.enter\",\n","        \"Key.esc\",\n","        \"Key.f1\",\n","        \"Key.f2\",\n","        \"Key.f3\",\n","        \"Key.f4\",\n","        \"Key.f5\",\n","        \"Key.f6\",\n","        \"Key.f7\",\n","        \"Key.f8\",\n","        \"Key.f9\",\n","        \"Key.f10\",\n","        \"Key.f11\",\n","        \"Key.f12\",\n","        \"Key.f13\",\n","        \"Key.f14\",\n","        \"Key.f15\",\n","        \"Key.f16\",\n","        \"Key.f17\",\n","        \"Key.f18\",\n","        \"Key.f19\",\n","        \"Key.f20\",\n","        \"Key.home\",\n","        \"Key.left\",\n","        \"Key.page_down\",\n","        \"Key.page_up\",\n","        \"Key.right\",\n","        \"Key.shift\",\n","        \"Key.shift\",\n","        \"Key.shift_r\",\n","        \"Key.space\",\n","        \"Key.tab\",\n","        \"Key.up\",\n","        \"Key.media_play_pause\",\n","        \"Key.media_volume_mute\",\n","        \"Key.media_volume_down\",\n","        \"Key.media_volume_up\",\n","        \"Key.media_previous\",\n","        \"Key.media_next\",\n","    ]\n","    \n","    length = (len(action_types)\n","            + len(keys)\n","            + len(mouse_buttons)\n","            + 1 # mouse pressed\n","            + 4 * mouse_resolution) #,\n","#             1)\n","\n","    @staticmethod\n","    def unshift_keycode(keycode: str) -> Union[str, None]:\n","        unshift_keycodes = {\n","            \"!\": \"1\",\n","            \"@\": \"2\",\n","            \"#\": \"3\",\n","            \"$\": \"4\",\n","            \"%\": \"5\",\n","            \"^\": \"6\",\n","            \"&\": \"7\",\n","            \"*\": \"8\",\n","            \"(\": \"9\",\n","            \")\": \"0\",\n","            \"_\": \"-\",\n","            \"+\": \"=\",\n","            \"{\": \"[\",\n","            \"}\": \"]\",\n","            \"|\": \"\\\\\",\n","            \":\": \";\",\n","            '\"': \"'\",\n","            \"<\": \",\",\n","            \">\": \".\",\n","            \"?\": \"/\",\n","            \"~\": \"`\",\n","        }\n","        ctrl_keycodes = {\n","            \"\\x01\": \"a\",\n","            \"\\x02\": \"b\",\n","            \"\\x03\": \"c\",\n","            \"\\x04\": \"d\",\n","            \"\\x05\": \"e\",\n","            \"\\x06\": \"f\",\n","            \"\\x07\": \"g\",\n","            \"\\x08\": \"h\",\n","            \"\\t\": \"i\",\n","            \"\\n\": \"j\",\n","            \"\\x0b\": \"k\",\n","            \"\\x0c\": \"l\",\n","            \"\\r\": \"m\",\n","            \"\\x0e\": \"n\",\n","            \"\\x0f\": \"o\",\n","            \"\\x10\": \"p\",\n","            \"\\x11\": \"q\",\n","            \"\\x12\": \"r\",\n","            \"\\x13\": \"s\",\n","            \"\\x14\": \"t\",\n","            \"\\x15\": \"u\",\n","            \"\\x16\": \"v\",\n","            \"\\x17\": \"w\",\n","            \"\\x18\": \"x\",\n","            \"\\x19\": \"y\",\n","            \"\\x1a\": \"z\",\n","            \"<219>\": \"[\",\n","            \"<221>\": \"]\",\n","            \"<189>\": \"-\",\n","            \"<187>\": \"=\",\n","            \"<192>\": \"`\",\n","            \"<48>\": \"0\",\n","            \"<49>\": \"1\",\n","            \"<50>\": \"2\",\n","            \"<51>\": \"3\",\n","            \"<52>\": \"4\",\n","            \"<53>\": \"5\",\n","            \"<54>\": \"6\",\n","            \"<55>\": \"7\",\n","            \"<56>\": \"8\",\n","            \"<57>\": \"9\",\n","            \"<220>\": \"\\\\\",\n","            \"<186>\": \";\",\n","            \"<222>\": \"'\",\n","            \"<188>\": \",\",\n","            \"<190>\": \".\",\n","            \"<191>\": \"/\",\n","        }\n","        keycode = unshift_keycodes.get(keycode, ctrl_keycodes.get(keycode, keycode))\n","        # still, this is something out of concern.\n","        if keycode.startswith(\"<\") and keycode.endswith(\">\"):\n","            print(\"Discarding unconvertable keycode: %s\" % keycode)\n","            # keycode = pynput.keyboard.KeyCode(int(keycode[1:-1]))\n","            return\n","        return keycode\n","\n","    @staticmethod\n","    def uncover_keycode(keycode: str) -> Union[str, None]:\n","        if not keycode.startswith(\"Key.\"):\n","            keycode_converted = HIDActionBase.unshift_keycode(\n","                keycode\n","                if keycode.startswith(\"<\") and keycode.endswith(\">\")\n","                else ast.literal_eval(keycode)\n","            )\n","            return keycode_converted\n","            # this could be None.\n","            # when this is None, simply skip this code. do not end the conversion. skip it.\n","        else:\n","            return keycode\n","\n","\n","class HIDAction(BaseModel, HIDActionBase):\n","    # static method: from_action\n","    # static method: from_ndarray\n","    # instance method: to_ndarray\n","    # instance method: to_action\n","    max_x: int\n","    max_y: int\n","    action_type: Union[\n","        Literal[\"key_press\"],  # [\"key_press\", \"'w'\"]\n","        Literal[\"key_release\"],  # [\"key_release\", \"'r'\"]\n","        Literal[\n","            \"mouse_move\"\n","        ],  # [\"mouse_move\", [176.7734375, 580.40625]], \"timeStamp\": 1680247557.125498}\n","        Literal[\n","            \"mouse_click\"\n","        ],  # [\"mouse_click\", [176.7734375, 580.40625, \"Button.left\", true]]\n","        Literal[\"mouse_scroll\"],  # [\"mouse_scroll\", [938.76171875, 318.75, 0, 0]]\n","#         None,  # end_of_action\n","    ] # you need to specify this.\n","    key: Union[\n","        Literal[\"\"\"','\"\"\"],\n","        Literal[\"\"\"'.'\"\"\"],\n","        Literal[\"\"\"'/'\"\"\"],\n","        Literal[\"\"\"';'\"\"\"],\n","        Literal[\"\"\"\\\"'\\\"\"\"\"],\n","        Literal[\"\"\"'['\"\"\"],\n","        Literal[\"\"\"']'\"\"\"],\n","        Literal[\"\"\"'\\\\'\"\"\"],\n","        Literal[\"\"\"'='\"\"\"],\n","        Literal[\"\"\"'-'\"\"\"],\n","        Literal[\"\"\"'0'\"\"\"],\n","        Literal[\"\"\"'9'\"\"\"],\n","        Literal[\"\"\"'8'\"\"\"],\n","        Literal[\"\"\"'7'\"\"\"],\n","        Literal[\"\"\"'6'\"\"\"],\n","        Literal[\"\"\"'5'\"\"\"],\n","        Literal[\"\"\"'4'\"\"\"],\n","        Literal[\"\"\"'3'\"\"\"],\n","        Literal[\"\"\"'2'\"\"\"],\n","        Literal[\"\"\"'1'\"\"\"],\n","        Literal[\"\"\"'`'\"\"\"],\n","        Literal[\"\"\"'a'\"\"\"],\n","        Literal[\"\"\"'b'\"\"\"],\n","        Literal[\"\"\"'c'\"\"\"],\n","        Literal[\"\"\"'d'\"\"\"],\n","        Literal[\"\"\"'e'\"\"\"],\n","        Literal[\"\"\"'f'\"\"\"],\n","        Literal[\"\"\"'g'\"\"\"],\n","        Literal[\"\"\"'h'\"\"\"],\n","        Literal[\"\"\"'i'\"\"\"],\n","        Literal[\"\"\"'j'\"\"\"],\n","        Literal[\"\"\"'k'\"\"\"],\n","        Literal[\"\"\"'l'\"\"\"],\n","        Literal[\"\"\"'m'\"\"\"],\n","        Literal[\"\"\"'n'\"\"\"],\n","        Literal[\"\"\"'o'\"\"\"],\n","        Literal[\"\"\"'p'\"\"\"],\n","        Literal[\"\"\"'q'\"\"\"],\n","        Literal[\"\"\"'r'\"\"\"],\n","        Literal[\"\"\"'s'\"\"\"],\n","        Literal[\"\"\"'t'\"\"\"],\n","        Literal[\"\"\"'u'\"\"\"],\n","        Literal[\"\"\"'v'\"\"\"],\n","        Literal[\"\"\"'w'\"\"\"],\n","        Literal[\"\"\"'x'\"\"\"],\n","        Literal[\"\"\"'y'\"\"\"],\n","        Literal[\"\"\"'z'\"\"\"],\n","        Literal[\"Key.alt\"],\n","        Literal[\"Key.alt\"],\n","        Literal[\"Key.alt_r\"],\n","        Literal[\"Key.alt_r\"],\n","        Literal[\"Key.backspace\"],\n","        Literal[\"Key.caps_lock\"],\n","        Literal[\"Key.cmd\"],\n","        Literal[\"Key.cmd\"],\n","        Literal[\"Key.cmd_r\"],\n","        Literal[\"Key.ctrl\"],\n","        Literal[\"Key.ctrl\"],\n","        Literal[\"Key.ctrl_r\"],\n","        Literal[\"Key.delete\"],\n","        Literal[\"Key.down\"],\n","        Literal[\"Key.end\"],\n","        Literal[\"Key.enter\"],\n","        Literal[\"Key.esc\"],\n","        Literal[\"Key.f1\"],\n","        Literal[\"Key.f2\"],\n","        Literal[\"Key.f3\"],\n","        Literal[\"Key.f4\"],\n","        Literal[\"Key.f5\"],\n","        Literal[\"Key.f6\"],\n","        Literal[\"Key.f7\"],\n","        Literal[\"Key.f8\"],\n","        Literal[\"Key.f9\"],\n","        Literal[\"Key.f10\"],\n","        Literal[\"Key.f11\"],\n","        Literal[\"Key.f12\"],\n","        Literal[\"Key.f13\"],\n","        Literal[\"Key.f14\"],\n","        Literal[\"Key.f15\"],\n","        Literal[\"Key.f16\"],\n","        Literal[\"Key.f17\"],\n","        Literal[\"Key.f18\"],\n","        Literal[\"Key.f19\"],\n","        Literal[\"Key.f20\"],\n","        Literal[\"Key.home\"],\n","        Literal[\"Key.left\"],\n","        Literal[\"Key.page_down\"],\n","        Literal[\"Key.page_up\"],\n","        Literal[\"Key.right\"],\n","        Literal[\"Key.shift\"],\n","        Literal[\"Key.shift\"],\n","        Literal[\"Key.shift_r\"],\n","        Literal[\"Key.space\"],\n","        Literal[\"Key.tab\"],\n","        Literal[\"Key.up\"],\n","        Literal[\"Key.media_play_pause\"],\n","        Literal[\"Key.media_volume_mute\"],\n","        Literal[\"Key.media_volume_down\"],\n","        Literal[\"Key.media_volume_up\"],\n","        Literal[\"Key.media_previous\"],\n","        Literal[\"Key.media_next\"],\n","        None,\n","    ] = None\n","\n","    mouse_button: Union[\n","        Literal[\"Button.left\"], Literal[\"Button.middle\"], Literal[\"Button.right\"], None\n","    ] = None\n","    mouse_pressed: Union[bool, None] = None\n","    x: Union[float, None] = None\n","    y: Union[float, None] = None\n","    dx: Union[float, None] = None\n","    dy: Union[float, None] = None\n","\n","    @validator(\"max_x\", \"max_y\")\n","    def greater_than_zero(cls, v):\n","        assert type(v) == int\n","        assert v > 0\n","        return v\n","\n","    @validator(\"action_type\")\n","    def action_type_within_action_types(cls, v):\n","        if v:\n","            assert v in HIDActionBase.action_types\n","        return v\n","\n","    @validator(\"key\")\n","    def key_within_keys(cls, v):\n","        if v:\n","            assert v in HIDActionBase.keys\n","        return v\n","\n","    @validator(\"mouse_button\")\n","    def mouse_button_within_mouse_buttons(cls, v):\n","        if v:\n","            assert v in HIDActionBase.mouse_buttons\n","        return v\n","\n","    @validator(\"mouse_pressed\")\n","    def mouse_pressed_type_check(cls, v):\n","        if v:\n","            assert type(v) == bool\n","        return v\n","\n","    @staticmethod\n","    def from_action_json(action_json: list, max_x: int, max_y: int):\n","        action_type = action_json[0]\n","        action_args = action_json[1]\n","\n","        construct_args = dict(max_x=max_x, max_y=max_y, action_type=action_type)\n","\n","        if action_type == \"key_press\":\n","            assert action_args in HIDActionBase.keys\n","\n","            construct_args.update(dict(key=action_args))\n","        elif action_type == \"key_release\":\n","            assert action_args in HIDActionBase.keys\n","\n","            construct_args.update(dict(key=action_args))\n","        elif action_type == \"mouse_move\":\n","            assert action_args[0] >= 0 and action_args[0] <= max_x\n","            assert action_args[1] >= 0 and action_args[1] <= max_y\n","\n","            construct_args.update(dict(x=action_args[0], y=action_args[1]))\n","        elif action_type == \"mouse_click\":\n","            assert action_args[0] >= 0 and action_args[0] <= max_x\n","            assert action_args[1] >= 0 and action_args[1] <= max_y\n","            assert action_args[2] in HIDActionBase.mouse_buttons\n","            assert type(action_args[3]) == bool\n","\n","            construct_args.update(\n","                dict(\n","                    x=action_args[0],\n","                    y=action_args[1],\n","                    mouse_button=action_args[2],\n","                    mouse_pressed=action_args[3],\n","                )\n","            )\n","        elif action_type == \"mouse_scroll\":\n","            assert action_args[0] >= 0 and action_args[0] <= max_x\n","            assert action_args[1] >= 0 and action_args[1] <= max_y\n","            assert action_args[2] >= -max_x and action_args[2] <= max_x\n","            assert action_args[3] >= -max_y and action_args[3] <= max_y\n","\n","            construct_args.update(\n","                dict(\n","                    x=action_args[0],\n","                    y=action_args[1],\n","                    dx=action_args[2],\n","                    dy=action_args[3],\n","                )\n","            )\n","        else:\n","            raise Exception(\n","                \"Unknown action type: %s\\naction args: %s\" % (action_type, action_args)\n","            )\n","\n","        mHIDAction = HIDAction(**construct_args)\n","        return mHIDAction\n","\n","    @staticmethod\n","    def from_ndarray(ndarray: np.ndarray, max_x: int, max_y: int):\n","        assert ndarray.shape == (\n","            HIDActionBase.length,\n","        )\n","        cursor = 0\n","\n","        action_type_ndarray = ndarray[cursor : cursor + len(HIDActionBase.action_types)]\n","        cursor += len(HIDActionBase.action_types)\n","        action_type_index = np.argmax(action_type_ndarray)\n","        action_type = HIDActionBase.action_types[action_type_index]\n","        del action_type_ndarray\n","        del action_type_index\n","\n","        construct_args = dict(max_x=max_x, max_y=max_y, action_type=action_type)\n","\n","        if action_type:\n","            key_ndarray = ndarray[cursor : cursor + len(HIDActionBase.keys)]\n","            cursor += len(HIDActionBase.keys)\n","            key_index = np.argmax(key_ndarray)\n","            key = HIDActionBase.keys[key_index]\n","            del key_ndarray\n","            del key_index\n","\n","            mouse_button_ndarray = ndarray[\n","                cursor : cursor + len(HIDActionBase.mouse_buttons)\n","            ]\n","            cursor += len(HIDActionBase.mouse_buttons)\n","            mouse_button_index = np.argmax(mouse_button_ndarray)\n","            mouse_button = HIDActionBase.mouse_buttons[mouse_button_index]\n","            del mouse_button_ndarray\n","            del mouse_button_index\n","\n","            mouse_pressed_ndarray = ndarray[cursor : cursor + 1]\n","            cursor += 1\n","            mouse_pressed = bool(mouse_pressed_ndarray[0][0])\n","            del mouse_pressed_ndarray\n","\n","            x_ndarray = ndarray[cursor : cursor + HIDActionBase.mouse_resolution]\n","            cursor += HIDActionBase.mouse_resolution\n","            x_index = np.argmax(x_ndarray)\n","            x = (x_index / HIDActionBase.mouse_resolution) * max_x\n","            del x_ndarray\n","            del x_index\n","\n","            y_ndarray = ndarray[cursor : cursor + HIDActionBase.mouse_resolution]\n","            cursor += HIDActionBase.mouse_resolution\n","            y_index = np.argmax(y_ndarray)\n","            y = (y_index / HIDActionBase.mouse_resolution) * max_y\n","            del y_ndarray\n","            del y_index\n","\n","            dx_ndarray = ndarray[cursor : cursor + HIDActionBase.mouse_resolution]\n","            cursor += HIDActionBase.mouse_resolution\n","            dx_index = np.argmax(dx_ndarray)\n","            dx = (dx_index / HIDActionBase.mouse_resolution) * 2 * max_x - max_x\n","            del dx_ndarray\n","            del dx_index\n","\n","            dy_ndarray = ndarray[cursor : cursor + HIDActionBase.mouse_resolution]\n","            cursor += HIDActionBase.mouse_resolution\n","            dy_index = np.argmax(dy_ndarray)\n","            dy = (dy_index / HIDActionBase.mouse_resolution) * 2 * max_y - max_y\n","            del dy_ndarray\n","            del dy_index\n","\n","            if action_type == \"key_press\":\n","                construct_args.update(dict(key=key))\n","            elif action_type == \"key_release\":\n","                construct_args.update(dict(key=key))\n","            elif action_type == \"mouse_move\":\n","                construct_args.update(dict(x=x, y=y))\n","            elif action_type == \"mouse_click\":\n","                construct_args.update(\n","                    dict(\n","                        x=x, y=y, mouse_button=mouse_button, mouse_pressed=mouse_pressed\n","                    )\n","                )\n","            elif action_type == \"mouse_scroll\":\n","                construct_args.update(dict(x=x, y=y, dx=dx, dy=dy))\n","        else:\n","            pass\n","\n","        del cursor\n","\n","        mHIDAction = HIDAction(**construct_args)\n","        return mHIDAction\n","\n","    def round_within(self, number: Union[int, float], number_name: str) -> int:\n","        result = round(number)\n","        if result > self.mouse_resolution:\n","            print(f\"Warning: {number_name} overflow\")\n","            print(f\"Value {result} greater than {self.mouse_resolution}\")\n","            return self.mouse_resolution\n","        elif result < 0:\n","            print(f\"Warning: {number_name} overflow\")\n","            print(f\"Value {result} smaller than 0\")\n","            return 0\n","        return result\n","\n","    def to_ndarray(\n","        self,\n","    ) -> np.ndarray:\n","        action_type_ndarray = np.zeros((len(self.action_types), 1))\n","        action_type_ndarray[self.action_types.index(self.action_type)] = 1\n","\n","        key_ndarray = np.zeros((len(self.keys), 1))\n","        if self.key:\n","            key_ndarray[self.keys.index(self.key)] = 1\n","\n","        mouse_button_ndarray = np.zeros((len(self.mouse_buttons), 1))\n","        if self.mouse_button:\n","            mouse_button_ndarray[self.mouse_buttons.index(self.mouse_button)] = 1\n","\n","        mouse_pressed_array = np.zeros((1, 1))\n","        if self.mouse_pressed:\n","            mouse_pressed_array[0] = 1\n","\n","        x_ndarray = np.zeros((self.mouse_resolution, 1))\n","        if self.x:\n","            x_ndarray[\n","                self.round_within(self.mouse_resolution * self.x / self.max_x, \"X\")\n","            ] = 1\n","\n","        y_ndarray = np.zeros((self.mouse_resolution, 1))\n","        if self.y:\n","            y_ndarray[\n","                self.round_within(self.mouse_resolution * self.y / self.max_y, \"Y\")\n","            ] = 1\n","\n","        dx_ndarray = np.zeros((self.mouse_resolution, 1))\n","        if self.dx:\n","            dx_ndarray[\n","                self.round_within(\n","                    self.mouse_resolution * (self.dx + self.max_x) / (2 * self.max_x),\n","                    \"DX\",\n","                )\n","            ] = 1\n","\n","        dy_ndarray = np.zeros((self.mouse_resolution, 1))\n","        if self.dy:\n","            dy_ndarray[\n","                self.round_within(\n","                    self.mouse_resolution * (self.dy + self.max_y) / (2 * self.max_y),\n","                    \"DY\",\n","                )\n","            ] = 1\n","\n","        ndarray = np.concatenate(\n","            [\n","                action_type_ndarray,\n","                key_ndarray,\n","                mouse_button_ndarray,\n","                mouse_pressed_array,\n","                x_ndarray,\n","                y_ndarray,\n","                dx_ndarray,\n","                dy_ndarray,\n","            ]\n","        )\n","        return ndarray\n","\n","    def to_action_json(\n","        self,\n","    ) -> Union[list, None]:\n","        action_type = self.action_type\n","        if action_type:\n","            if action_type == \"key_press\":\n","                assert self.key in self.keys\n","\n","                action_args = self.key\n","            elif action_type == \"key_release\":\n","                assert self.key in self.keys\n","\n","                action_args = self.key\n","            elif action_type == \"mouse_move\":\n","                assert self.x >= 0 and self.x <= self.max_x\n","                assert self.y >= 0 and self.y <= self.max_y\n","\n","                action_args = [self.x, self.y]\n","            elif action_type == \"mouse_click\":\n","                assert self.x >= 0 and self.x <= self.max_x\n","                assert self.y >= 0 and self.y <= self.max_y\n","                assert self.mouse_button in self.mouse_buttons\n","                assert type(self.mouse_pressed) == bool\n","\n","                action_args = [self.x, self.y, self.mouse_button, self.mouse_pressed]\n","            elif action_type == \"mouse_scroll\":\n","                assert self.x >= 0 and self.x <= self.max_x\n","                assert self.y >= 0 and self.y <= self.max_y\n","                assert self.dx >= -self.max_x and self.dx <= self.max_x\n","                assert self.dy >= -self.max_y and self.dy <= self.max_y\n","\n","                action_args = [self.x, self.y, self.dx, self.dy]\n","            else:\n","                raise Exception(\"Unknown action_type: %s\" % action_type)\n","            action_json = [action_type, action_args]\n","        else:\n","            action_json = None\n","        return action_json\n"]},{"cell_type":"markdown","metadata":{},"source":["## read data from path"]},{"cell_type":"code","execution_count":158,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:37.690481Z","iopub.status.busy":"2023-04-22T15:36:37.690097Z","iopub.status.idle":"2023-04-22T15:36:37.718167Z","shell.execute_reply":"2023-04-22T15:36:37.717059Z","shell.execute_reply.started":"2023-04-22T15:36:37.690445Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["PERSPECTIVE: 1280 800\n","TIMESTEP: 0.03\n"]}],"source":["\n","import json\n","\n","with open(metadata_file_path, \"r\") as f:\n","    data = json.loads(f.read())\n","\n","# print(data.keys())\n","# ['screenshot_and_actions', 'perspective_size', 'timestep']\n","\n","perspective_width, perspective_height = data[\"perspective_size\"]\n","timestep = data[\"timestep\"]\n","\n","print(\"PERSPECTIVE:\", perspective_width, perspective_height)\n","print(\"TIMESTEP:\", timestep)\n","\n","import re\n","\n","class VideoCaptureContextManager:\n","    def __init__(self, videoPath):\n","        self.videoPath = videoPath\n","        \n","    def __enter__(self):\n","        print(\"Entering the context...\")\n","        self.cap = cv2.VideoCapture(self.videoPath)\n","        return self.cap\n","\n","    def __exit__(self, exc_type, exc_value, exc_tb):\n","        try:\n","            self.cap.release()\n","        finally:\n","            import gc\n","            gc.collect()\n","            print(\"Leaving the context...\")\n","        #  print(exc_type, exc_value, exc_tb, sep=\"\\n\")\n"]},{"cell_type":"markdown","metadata":{},"source":["## test and define the model"]},{"cell_type":"code","execution_count":159,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:37.722030Z","iopub.status.busy":"2023-04-22T15:36:37.721294Z","iopub.status.idle":"2023-04-22T15:36:37.727214Z","shell.execute_reply":"2023-04-22T15:36:37.725949Z","shell.execute_reply.started":"2023-04-22T15:36:37.721986Z"},"trusted":true},"outputs":[],"source":["import torch\n","import torchvision"]},{"cell_type":"code","execution_count":160,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:37.729895Z","iopub.status.busy":"2023-04-22T15:36:37.729155Z","iopub.status.idle":"2023-04-22T15:36:39.414488Z","shell.execute_reply":"2023-04-22T15:36:39.413142Z","shell.execute_reply.started":"2023-04-22T15:36:37.729853Z"},"trusted":true},"outputs":[],"source":["vit_model = torchvision.models.vit_b_16(pretrained=True)"]},{"cell_type":"code","execution_count":161,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:39.416888Z","iopub.status.busy":"2023-04-22T15:36:39.416518Z","iopub.status.idle":"2023-04-22T15:36:39.422344Z","shell.execute_reply":"2023-04-22T15:36:39.421161Z","shell.execute_reply.started":"2023-04-22T15:36:39.416850Z"},"trusted":true},"outputs":[],"source":["\n","# use einops.pack and einops.unpack\n","\n","# def size_splits(tensor, split_sizes, dim=0):\n","#     \"\"\"Splits the tensor according to chunks of split_sizes.\n","    \n","#     Arguments:\n","#         tensor (Tensor): tensor to split.\n","#         split_sizes (list(int)): sizes of chunks\n","#         dim (int): dimension along which to split the tensor.\n","#     \"\"\"\n","#     if dim < 0:\n","#         dim += tensor.dim()\n","    \n","#     dim_size = tensor.size(dim)\n","#     if dim_size != torch.sum(torch.Tensor(split_sizes)):\n","#         raise KeyError(\"Sum of split sizes exceeds tensor dim\")\n","    \n","#     splits = torch.cumsum(torch.Tensor([0] + split_sizes), dim=0)[:-1]\n","\n","#     return tuple(tensor.narrow(int(dim), int(start), int(length)) \n","#         for start, length in zip(splits, split_sizes))"]},{"cell_type":"code","execution_count":162,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:39.424847Z","iopub.status.busy":"2023-04-22T15:36:39.424016Z","iopub.status.idle":"2023-04-22T15:36:51.458281Z","shell.execute_reply":"2023-04-22T15:36:51.456926Z","shell.execute_reply.started":"2023-04-22T15:36:39.424810Z"},"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Requirement already satisfied: pydantic-numpy in /opt/conda/lib/python3.7/site-packages (1.3.0)\n","Requirement already satisfied: pydantic in /opt/conda/lib/python3.7/site-packages (from pydantic-numpy) (1.10.4)\n","Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from pydantic-numpy) (1.21.6)\n","Requirement already satisfied: typing-extensions>=4.2.0 in /opt/conda/lib/python3.7/site-packages (from pydantic->pydantic-numpy) (4.4.0)\n","\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n","\u001b[0m"]}],"source":["!pip3 install pydantic-numpy\n","from pydantic_numpy import NDArray"]},{"cell_type":"code","execution_count":164,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:51.500838Z","iopub.status.busy":"2023-04-22T15:36:51.500431Z","iopub.status.idle":"2023-04-22T15:36:51.537344Z","shell.execute_reply":"2023-04-22T15:36:51.536192Z","shell.execute_reply.started":"2023-04-22T15:36:51.500801Z"},"trusted":true},"outputs":[],"source":["# notice: when in online mode only image will be backpropagated.\n","# like using some upside down mirror.\n","\n","\n","class CustomModel(torch.nn.Module):\n","    def __init__(self, vit_model, hidden_size_vit=1000, vit_block_size=228):\n","#     def __init__(self, rwkv_model, vit_model, tokenizer, hidden_size_rwkv, hidden_size_vit, output_size, vit_times = 4, vit_block_size=228):\n","        super(CustomModel, self).__init__()\n","#         self.rwkv_model = rwkv_model # processing language, generate actions.\n","        self.vit_model = vit_model\n","        self.vit_block_size = vit_block_size # this is default.\n","#         self.vit_times = vit_times\n","        \n","        # seq2seq alike.\n","#         self.hidden_size = hidden_size_rwkv+hidden_size_vit\n","        self.hidden_size = hidden_size_vit\n","    \n","        self.HIDEncoder = torch.nn.Linear(HIDActionBase.length, 1000)\n","        self.HIDDecoder = torch.nn.Linear(1000, HIDActionBase.length) # use torch.where or something \n","        # sparse matrix?\n","        \n","        # some magic going elsewhere.\n","        self.ViTDecoder = torch.nn.Conv2d(in_channels=9, out_channels=3, kernel_size=22, stride=4, dilation=5) # n c h w\n","    \n","        # FIX 13: change 1000 to ConsciousBase.data_type_length+ConsciousBase.special_token_length+1000\n","        io_h_size=ConsciousBase.data_type_length+ConsciousBase.special_token_length+1000\n","        self.rnn = torch.nn.LSTM(input_size=io_h_size, hidden_size = io_h_size, batch_first=True)\n","\n","        # use tensor.\n","    def forward(self, conscious_stream:torch.Tensor, target_output:Union[None, torch.Tensor]=None):\n","        # if have target, and our datatype bits are wrong, we only optimize the datatype bits, making image/action bits identical\n","        # otherwise just calculate all bits from the model.\n","        \n","        # conscious_stream: [batch_size, (ConsciousBase.length) data_type+special_tokens+image_bits+action_bits]\n","        # you need another pydantic model for it. \n","        \n","        # BUG 9: input shape error\n","        # FIX 9: check input and output shape\n","#         print(conscious_stream.shape, ConsciousBase.length, )\n","#         if target_output is not None:\n","#             print(target_output.shape)\n","        batch_size, seq_length, dim_block = conscious_stream.shape\n","        assert dim_block == ConsciousBase.length\n","        \n","        conscious_stream_reshaped = einops.rearrange(conscious_stream, \"b s d -> (b s) d\")\n","        \n","        datatype_bits, special_bits, image_bits, action_bits = einops.unpack(conscious_stream_reshaped, [[s] for s in ConsciousBase.split_sizes],\"b *\")\n","#         datatype_bits, special_bits, image_bits, action_bits = size_splits(conscious_stream_reshaped, ConsciousBase.split_sizes,dim=1)\n","#         desired_size = self.vit_block_size*self.vit_times\n","\n","        datatypes = torch.argmax(datatype_bits, dim=1)\n","        image_indexs = datatypes == 0\n","        action_indexs = datatypes == 1\n","        \n","        special_tokens = torch.argmax(special_bits, dim=1)\n","        image_newline_indexs = special_tokens == 0\n","        image_end_indexs = special_tokens == 1\n","        action_end_indexs = special_tokens == 2\n","        nop_indexs = special_tokens == 3\n","        \n","#         4+2+1000\n","        \n","        # you are gonna take actions.\n","        # prepare some zeros.\n","        \n","        # BUG 10: len() on int\n","        # FIX 10: find and fix incorrect len() calls\n","        batched_rnn_input = torch.zeros((batch_size*seq_length, ConsciousBase.data_type_length+ConsciousBase.special_token_length+1000))\n","        \n","        batched_rnn_input[image_indexs, 0] = 1\n","        batched_rnn_input[action_indexs, 1] = 1\n","        \n","        # BUG 11: indexs not defined\n","        # FIX 11: indexes -> indexs\n","        batched_rnn_input[image_newline_indexs, 2] = 1\n","        batched_rnn_input[image_end_indexs, 3] = 1\n","        batched_rnn_input[action_end_indexs, 4] = 1\n","        batched_rnn_input[nop_indexs, 5] = 1\n","        \n","        # process this.\n","        datatype_and_special_token_length = ConsciousBase.data_type_length + ConsciousBase.special_token_length\n","        \n","        # BUG 12: unannotated unknown axes in einops.rearrange\n","        # FIX 12: annotate these axes\n","        selected_image_bits = image_bits[image_indexs, :]\n","        transformed_image_bits = einops.rearrange(selected_image_bits, \"b (c h w) -> b c h w\", h = ConsciousBase.image_dim, w = ConsciousBase.image_dim, c = ConsciousBase.image_channels)\n","        processed_image_bits = self.vit_model(transformed_image_bits)\n","        batched_rnn_input[image_indexs, datatype_and_special_token_length:] = processed_image_bits\n","        \n","        selected_action_bits = action_bits[action_indexs, :]\n","        processed_action_bits = self.HIDEncoder(selected_action_bits)\n","        batched_rnn_input[action_indexs, datatype_and_special_token_length:] = processed_action_bits\n","        \n","        batched_rnn_input_reshaped = einops.rearrange(batched_rnn_input, \"(b s) d -> b s d\", b = batch_size, s = seq_length)\n","        \n","        # BUG 13: mismatched shape for rnn\n","        _, (h1, c1) = self.rnn(batched_rnn_input_reshaped)\n","        \n","        # shape of h1: [batch_size, dim_block]\n","        \n","        if target_output is not None:\n","            target_datatype_bits, target_special_bits, target_image_bits, target_action_bits = einops.unpack(target_output, [[s] for s in ConsciousBase.split_sizes], \"b *\")\n","        \n","        output_datatype_bits, output_special_bits, output_data_bits = einops.unpack(einops.rearrange(h1, \"b s d -> (b s) d\"), [[s] for s in [ConsciousBase.data_type_length, ConsciousBase.special_token_length, 1000]] , \"b *\")\n","#         output_datatype_bits, output_special_bits, output_data_bits = size_splits(h1, [ConsciousBase.data_type_length, ConsciousBase.special_token_length, 1000] ,dim=1)\n","        output_datatypes = torch.argmax(output_datatype_bits, dim=1)\n","        \n","        output_image_indexs = output_datatypes == 0\n","        output_action_indexs = output_datatypes == 1\n","        \n","        if target_output is not None:\n","            target_output_datatypes = torch.argmax(target_datatype_bits, dim=1)\n","            \n","            target_output_image_indexs = target_output_datatypes == 0\n","            target_output_action_indexs = target_output_datatypes == 1\n","            \n","            common_output_image_indexs = torch.logical_and(target_output_image_indexs, output_image_indexs)\n","            common_output_action_indexs = torch.logical_and(target_output_action_indexs, output_action_indexs)\n","            \n","            common_output_indexs = torch.logical_or(common_output_image_indexs, common_output_action_indexs)\n","            \n","            target_exclusive_output_image_indexs = torch.logical_and(target_output_image_indexs, torch.logical_not(output_image_indexs))\n","            target_exclusive_output_action_indexs = torch.logical_and(target_output_action_indexs, torch.logical_not(output_action_indexs))\n","            \n","            target_exclusive_output_indexs = torch.logical_or(target_exclusive_output_image_indexs, target_exclusive_output_action_indexs)\n","        \n","        if target_output is not None:\n","            selected_output_image_bits = output_data_bits[common_output_image_indexs, :]\n","        else:\n","            selected_output_image_bits = output_data_bits[output_image_indexs, :]\n","        processed_output_image_bits = einops.repeat(selected_output_image_bits, \"b d -> b d 9\")\n","        processed_output_image_bits = einops.einsum(processed_output_image_bits, processed_output_image_bits,\"b h c, b w c -> b c h w\")\n","        processed_output_image_bits = self.ViTDecoder(processed_output_image_bits)\n","        processed_output_image_bits = einops.rearrange(processed_output_image_bits, \"b c h w -> b (c h w)\")\n","        \n","        if target_output is not None:\n","            selected_output_action_bits = output_data_bits[common_output_action_indexs,:]\n","        else:\n","            selected_output_action_bits = output_data_bits[output_action_indexs,:]\n","        processed_output_action_bits = self.HIDDecoder(selected_output_action_bits)\n","        \n","        # preparing blank output\n","        output_0 = torch.zeros((batch_size, ConsciousBase.length))\n","        \n","        output_datatype_bits_0, output_special_token_bits_0, output_image_bits_0, output_action_bits_0 = einops.unpack(output_0, [[s] for s in ConsciousBase.split_sizes], \"b *\")\n","#         output_datatype_bits_0, output_special_token_bits_0, output_image_bits_0, output_action_bits_0 = size_splits(output, ConsciousBase.split_sizes, dim=1)\n","        \n","        if target_output is not None:\n","            output_datatype_bits_0[target_exclusive_output_indexs,:] = target_datatype_bits[target_exclusive_output_indexs,:]\n","            output_datatype_bits_0[common_output_indexs,:] = output_datatype_bits[common_output_indexs,:]\n","            \n","            output_special_token_bits_0[target_exclusive_output_indexs,:] = output_special_bits[target_exclusive_output_indexs,:]\n","            output_special_token_bits_0[common_output_indexs,:] = output_special_bits[common_output_indexs,:]\n","            \n","            output_special_token_bits_0[common_output_image_indexs,2] = 0\n","            output_special_token_bits_0[common_output_action_indexs,:2] = 0\n","            \n","            output_image_bits_0[target_exclusive_output_image_indexs,:] = target_image_bits[target_exclusive_output_image_indexs,:]\n","            output_action_bits_0[target_exclusive_output_action_indexs, :] = target_action_bits[target_exclusive_output_action_indexs,:]\n","        \n","            output_image_bits_0[common_output_image_indexs,:] = processed_output_image_bits\n","            output_action_bits_0[common_output_action_indexs, :] = processed_output_action_bits\n","        else:\n","            output_datatype_bits_0 = output_datatype_bits\n","            \n","            output_special_bits_0 = output_special_bits\n","            \n","            output_special_bits_0[output_image_indexs, 2] = 0\n","            output_special_bits_0[output_action_indexs, :2] = 0\n","        \n","            output_image_bits_0[output_image_indexs, :] = processed_output_image_bits\n","            output_action_bits_0[output_action_indexs, :] = processed_output_action_bits\n","        \n","        output, _ = einops.pack((output_datatype_bits_0, output_special_token_bits_0, output_image_bits_0, output_action_bits_0), \"b *\")\n","#         output = torch.concat((output_datatype_bits_0, output_special_token_bits_0, output_image_bits_0, output_action_bits_0), dim=1)\n","        \n","        return output"]},{"cell_type":"markdown","metadata":{},"source":["## training"]},{"cell_type":"code","execution_count":165,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:51.540131Z","iopub.status.busy":"2023-04-22T15:36:51.539555Z","iopub.status.idle":"2023-04-22T15:36:51.718848Z","shell.execute_reply":"2023-04-22T15:36:51.717347Z","shell.execute_reply.started":"2023-04-22T15:36:51.540084Z"},"trusted":true},"outputs":[],"source":["# train the model.\n","rnn_hidden_state = None # because this is needed for every session per video.\n","\n","# hidden_size_rwkv = hidden_size_vit = 1024\n","# output_size = # output size shall be identical to ConsciousBase.length\n","\n","model = CustomModel(vit_model)\n"]},{"cell_type":"code","execution_count":166,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:51.721251Z","iopub.status.busy":"2023-04-22T15:36:51.720840Z","iopub.status.idle":"2023-04-22T15:36:51.729547Z","shell.execute_reply":"2023-04-22T15:36:51.728329Z","shell.execute_reply.started":"2023-04-22T15:36:51.721212Z"},"trusted":true},"outputs":[],"source":["from torch.optim import Adam\n","# from torch.nn import BCELoss\n","# from torch.nn import MSELoss\n","from torch.nn import CrossEntropyLoss\n","lr = 0.00001\n","optimizer= Adam(model.parameters(), lr=lr)\n","\n","# loss_fn = BCELoss(reduction='none'\n","# loss_fn = MSELoss()\n","# loss_fn = CrossEntropyLoss(reduction='sum')\n","loss_fn = CrossEntropyLoss(reduction='mean')"]},{"cell_type":"code","execution_count":167,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:51.732296Z","iopub.status.busy":"2023-04-22T15:36:51.731684Z","iopub.status.idle":"2023-04-22T15:36:51.743964Z","shell.execute_reply":"2023-04-22T15:36:51.742603Z","shell.execute_reply.started":"2023-04-22T15:36:51.732240Z"},"trusted":true},"outputs":[],"source":["import cv2\n","\n","def resizeImage(im, desired_size):\n","    # im = cv2.imread(im_pth)\n","    old_size = im.shape[:2] # old_size is in (height, width) format\n","\n","    ratio = float(desired_size)/max(old_size)\n","    new_size = tuple([int(x*ratio) for x in old_size])\n","\n","    # new_size should be in (width, height) format\n","\n","    im = cv2.resize(im, (new_size[1], new_size[0]))\n","\n","    delta_w = desired_size - new_size[1]\n","    delta_h = desired_size - new_size[0]\n","    top, bottom = delta_h//2, delta_h-(delta_h//2)\n","    left, right = delta_w//2, delta_w-(delta_w//2)\n","\n","    color = [0, 0, 0]\n","    new_im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT,\n","        value=color)\n","    return new_im"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2023-04-22T15:36:51.749969Z","iopub.status.busy":"2023-04-22T15:36:51.748929Z"},"scrolled":true,"trusted":true},"outputs":[{"name":"stdout","output_type":"stream","text":["Entering the context...\n","\n","SCREENSHOT: {'timeStamp': 1680247551.452273, 'imagePath': 'screenshots/0.raw', 'imageSize': [2560, 1600]}\n","IMAGE INDEX: 0\n","LOSS?\n","tensor(1.5712e+08, grad_fn=<DivBackward1>)\n","LOSS?\n","tensor(63620880., grad_fn=<DivBackward1>)\n","ACTIONS: []\n","\n","SCREENSHOT: {'timeStamp': 1680247551.510298, 'imagePath': 'screenshots/1.raw', 'imageSize': [2560, 1600]}\n","IMAGE INDEX: 1\n","LOSS?\n","tensor(1.0702e+08, grad_fn=<DivBackward1>)\n"]}],"source":["# just for test.\n","# shall we do some backward?\n","\n","# consider merging multiple ViT converted images to batch them?\n","# from PIL import Image\n","\n","desired_size = 224*4\n","\n","# for every image it adds 16 conscious block in total.\n","# it could be a huge array\n","\n","# so we would append into some queue class, when hit the critical point, it will train.\n","from typing import Callable\n","\n","import gc\n","\n","class Trainer:\n","    def __init__(self, model, loss_fn, optimizer):\n","        self.model = model\n","        self.loss_fn = loss_fn\n","        self.optimizer = optimizer # shall be registered to model parameters.\n","        \n","    def step(self, batched_input, batched_output=None):\n","        # BUG 8: model forward keyword error\n","        # FIX 8: fixing keyword, adding default keyword argument\n","        model_output = self.model.forward(batched_input, target_output=batched_output)\n","        loss = self.loss_fn(model_output, batched_output)\n","        print(\"LOSS?\")\n","        print(loss)\n","        \n","        # this loss is incorrect. shall use some argmax stuff.\n","        # to ensure that this thing is the thing that we want.\n","        loss.backward()\n","        self.optimizer.step()\n","        self.optimizer.zero_grad()\n","        \n","\n","\n","class SequentialTrainingQueue:\n","    def __init__(self, context_length:int, batch_size:int, trainer:Trainer):\n","        self.context_length = context_length\n","        self.batch_size = batch_size\n","        \n","        self.trainer = trainer\n","        \n","        self.max_critical_length = self.context_length+self.batch_size\n","        self.min_critical_length = self.context_length+1\n","        \n","        self.consciousVectors = []\n","        \n","    def enqueue(self, consciousBlock:ConsciousBlock, clear:bool=False):\n","        # change that into some tensor first.\n","        \n","        # BUG 3: consciousBlock has tensor output but not numpy ndarray\n","        # FIX 3: find and replace all \"consciousBlock.to_nparray()\" with \"consciousBlock.to_tensor()\"\n","#         print(consciousBlock)\n","#         print(type(consciousBlock))\n","        consciousVector = consciousBlock.to_tensor()\n","        self.consciousVectors.append(consciousVector)\n","        if not clear:\n","            # BUG 5: no \"max_critical_point\"\n","            # FIX 5: replace it with \"max_critical_length\"\n","            if len(self.consciousVectors) == self.max_critical_length:\n","                self.train()\n","        else:\n","            self.clear()\n","    \n","    def train(self, clear:bool=False):\n","        # train the model and clear the queue.\n","        # size of queue before: self.context_length+self.batch_size (should be? at least geq to self.length+1)\n","        # size of queue after training: self.context_length\n","        \n","        if len(self.consciousVectors) >= self.min_critical_length:\n","            batch_size = len(self.consciousVectors) - self.context_length\n","            # BUG 6: missing self.\n","            # FIX 6: adding self.\n","            \n","            # BUG 7: torch.Tensor conversion error\n","            # FIX 7: change \"torch.Tensor([])\" into einops.pack\n","            \n","#             print(self.consciousVectors)\n","            batched_input, _ = einops.pack([\n","#             batched_input = torch.Tensor([\n","                    einops.pack(self.consciousVectors[i:i+self.context_length],'* d')[0] for i in range(batch_size)\n","#                     torch.Tensor(self.consciousVectors[i:i+self.context_length]) for i in range(batch_size)\n","                ],'* s d')\n","            batched_output, _ = einops.pack([\n","#             batched_output = torch.Tensor([\n","                    self.consciousVectors[self.context_length+i] for i in range(batch_size)\n","                ], '* d')\n","            self.trainer.step(batched_input, batched_output)\n","            \n","            if not clear:\n","                # now remove some elements.\n","                self.consciousVectors = self.consciousVectors[-self.context_length:]\n","            else:\n","                self.consciousVectors = []\n","        \n","    def clear(self):\n","        # check if anything left in queue. call at the end of queue.\n","        self.train(clear=True)\n","\n","# technically we would \"shuffle\" the dataset.\n","# but now, we just read it in sequence.\n","\n","# make it small.\n","context_length = 10\n","batch_size = 3\n","\n","trainer = Trainer(model, loss_fn, optimizer)\n","\n","sequentialTrainingQueue = SequentialTrainingQueue(context_length, batch_size, trainer)\n","\n","with VideoCaptureContextManager(videoPath) as cap:\n","    last_index = -1\n","    for screenshot_and_actions in data[\"screenshot_and_actions\"]:\n","        screenshot = screenshot_and_actions[\"screenshot\"]\n","        actions = screenshot_and_actions[\"actions\"]\n","        print()\n","        print(\"SCREENSHOT:\", screenshot)\n","        image_path = screenshot[\"imagePath\"]\n","        image_size = screenshot[\"imageSize\"]\n","        index = re.findall(r\"\\d+\", image_path)[0]\n","        index = int(index)\n","        print(\"IMAGE INDEX:\", index)\n","        \n","        encoded_actions = []\n","\n","        for action in actions:\n","            action_type, action_args = action[\"HIDEvent\"]\n","            if action_type in HIDActionBase.keyboard_action_types:\n","                action_args = HIDActionBase.uncover_keycode(action_args)\n","                if action_args is None:\n","                    print(\"Skipping\")\n","                    continue\n","            mHIDAction = HIDAction.from_action_json(\n","                [action_type, action_args],\n","                max_x=perspective_width,\n","                max_y=perspective_height,\n","            )  # related to mouse coordinates.\n","            mHIDActionNDArray = mHIDAction.to_ndarray()\n","            print(mHIDActionNDArray.shape)\n","\n","\n","        if index != last_index:\n","            ret, image = cap.read()\n","#             print(image.shape) # 400, 640, 3\n","# shall be the lowest quality.\n","            # loop through all actions in this area.\n","            image_resized = resizeImage(image, desired_size)\n","            image_reshaped = einops.rearrange(image_resized, \"h w c -> c h w\")\n","#             image_reshaped = np.rollaxis(image_resized, 2, 0) # (3, 896, 896)\n","            image_sliced = [\n","                image_reshaped[:,\\\n","                                                   x*224:(x+1)*224,\\\n","                                                   y*224:(y+1)*224\n","                                                  ] for x in range(4) for y in range(4)\n","            ] #) # c h w\n","            \n","            # IMAGE RESHAPED: (896, 3, 896)?\n","            # IMAGE RESHAPED: (896, 896, 3)\n","#             print('IMAGE RESHAPED:', image_reshaped.shape)\n","#             print('IMAGE SLICED:', image_sliced.shape)\n","#     (16, 3, 224, 224)\n","# hell?\n","            for index, im in enumerate(image_sliced):\n","                im = einops.rearrange(im, \"c h w -> (c h w)\")\n","                st = None\n","                if index == 15:\n","                    st = \"image_end\"\n","                elif index !=0 and (index+1)%4 == 0:\n","                    st = \"image_newline\"\n","                    \n","                # BUG 4: tuple\n","                # FIX 4: remove .to_tensor() method call\n","                consciousBlock = ConsciousBlock(data_type='image', special_token=st, image_data=im)\n","#                 print(consciousBlock)\n","                sequentialTrainingQueue.enqueue(consciousBlock)\n","#             last_output = torch.zeros(1, output_size)\n","            for index, EA in enumerate(encoded_actions):\n","                st = None\n","                if index+1 == len(encoded_actions):\n","                    st = \"action_end\"\n","                consciousBlock = ConsciousBlock(data_type = \"action\", special_token=st, action_data = EA)\n","                sequencialTrainingQueue.enqueue(consciousBlock)\n","                \n","#                 print('ACTION SHAPE?', EA.to_ndarray().shape) # (4111, 1)\n","#                 if last_output is not None:\n","#                     last_output = last_output.detach()\n","#                 if rnn_hidden_state is not None:\n","#                     rnn_hidden_state = (rnn_hidden_state[0].detach(), rnn_hidden_state[1].detach())\n","#                 last_output, rnn_hidden_state = model.forward(torch.tensor(image_sliced).float(), last_output, rnn_hidden_state)\n","# #                 output, rnn_hidden_state = model.forward(instruction, pil_image, rnn_hidden_state)\n","#                 print(\"OUTPUT:\", last_output.shape)\n","#                 print(\"HIDDEN_STATE:\", rnn_hidden_state[0].shape, rnn_hidden_state[1].shape) # used for next action, if possible.\n","#                 target = torch.tensor(EA.to_ndarray().reshape(1,-1)).float()\n","#                 print(\"TARGET SHAPE:\", target.shape)\n","#                 print(\"OUTPUT SHAPE:\", last_output.shape)\n","#                 # what the heck?\n","# #                 TARGET SHAPE: torch.Size([1, 4111])\n","# #                 OUTPUT SHAPE: torch.Size([1, 4111])\n","#                 print(target)\n","#                 print(last_output)\n","#                 with torch.enable_grad():\n","                \n","            del image\n","            del image_reshaped\n","            del image_resized\n","            last_index = index\n","        else:\n","            pass\n","        \n","        # no need to jump to index. it is monotonic.\n","        # image = read_image_from_video_at_given_index_as_ndarray(index, cap)\n","        # SCREENSHOT: {'timeStamp': 1680247598.852561, 'imagePath': 'screenshots/498.raw', 'imageSize': [2560, 1600]}\n","        # print(image.shape)  # (1600, 2560, 3)\n","        # width, height, channel\n","        # breakpoint()\n","        \n","        print(\"ACTIONS:\", actions)\n","        # keyboard action types: ['key_press', 'key_release']\n","        # mouse action types: ['mouse_move', 'mouse_click', 'mouse_scroll']\n","        # cv2.imshow(\"IMG\", image)\n","        # cv2.waitKey(0)        \n","        # del image\n","        \n","        # not deleting the image. keep it!\n","\n","        # now the data reader is complete. focus on the network design.\n","        # shall we?\n","sequentialTrainingQueue.clear()"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"}},"nbformat":4,"nbformat_minor":4}