simple.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a304c07b-37ad-467e-9c31-901d6e141ab0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Precompiling WebDataset [dfbb906c-54e0-4d57-86b9-79210db879da]\n",
      "└ @ Base loading.jl:1317\n"
     ]
    }
   ],
   "source": [
    "using WebDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4500c571-956f-4903-8f70-7db005c628c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDescriptor\n",
       "  sources: Array{String}((4,))\n",
       "  shuffle: Int64 1000\n",
       "  batchsize: Int64 32\n",
       "  decoding: Array{Pair{String, Function}}((8,))\n",
       "  augmenting: Array{Pair{String, Function}}((1,))\n",
       "  collating: Array{Pair{String, Function}}((2,))\n",
       "  csize: Int64 100\n",
       "  ntasks: Int64 4\n",
       "  verbose: Bool false\n",
       "  debug: Bool true\n"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "collating = [\n",
    "    \".cls\" => data->Array{Int}(data),\n",
    "    \"\" => x->x\n",
    "]\n",
    "\n",
    "desc = DatasetDescriptor(\n",
    "    sources=[\"test.tar\", \"test.tar\", \"test.tar\", \"test.tar\"],\n",
    "    shuffle=1000,\n",
    "    batchsize=32,\n",
    "    decoding=default_decoders,\n",
    "    collating=collating,\n",
    "    debug=true,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9da81fbd-dbf2-4f2c-8a21-8dda3a53cf48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source = \"test.tar\"\n",
      "batch[\"__key__\"] = [\"1170061\", \"0397087\", \"0639471\", \"1161000\", \"0682696\", \"0178175\", \"0445946\", \"0778404\", \"1144107\", \"0346311\", \"0500203\", \"0357943\", \"0416617\", \"1095227\", \"0733180\", \"0416786\", \"1130008\", \"0756769\", \"0322758\", \"0850099\", \"1133072\", \"0137079\", \"0951895\", \"1180873\", \"0531201\", \"0301434\", \"0233180\", \"0095821\", \"1082116\", \"0279075\", \"0152434\", \"1150390\"]\n",
      "batch[\"__key__\"] = [\"0546002\", \"1090055\", \"0838118\", \"1188011\", \"0031111\", \"0889275\", \"0666452\", \"1276629\", \"0049013\", \"0325256\", \"0395183\", \"1180685\", \"1079896\", \"0321910\", \"0714281\", \"0957334\", \"0253681\", \"0690600\", \"0739575\", \"0196620\", \"0601446\", \"0666746\", \"0467996\", \"1143642\", \"0852838\", \"1026607\", \"0095975\", \"0521264\", \"0255724\", \"0146122\", \"0684631\", \"0439116\"]\n",
      "batch[\"__key__\"] = [\"0402447\", \"0659195\", \"1170518\", \"1017792\", \"0471436\", \"1070677\", \"0370303\", \"0049346\", \"1067949\", \"0257216\", \"0587640\", \"0968119\", \"1139147\", \"0055308\", \"0953357\", \"0085896\", \"0575841\", \"0456061\", \"1028175\", \"0413149\", \"0164934\", \"0209750\", \"1159667\", \"0417770\", \"1166054\", \"0341154\", \"0396276\", \"0388746\", \"0227704\", \"0655984\", \"0792934\", \"0451421\"]\n",
      "batch[\"__key__\"] = [\"0841284\", \"0092600\", \"1183063\", \"0317191\", \"1135628\", \"0124163\", \"0112029\", \"0405008\", \"0103316\", \"0019009\", \"1089682\", \"1191438\", \"0899717\", \"0783279\", \"1271024\", \"0936383\", \"0274236\", \"1027408\", \"0106520\", \"0819786\", \"0897787\", \"0063481\", \"0442676\", \"0029327\", \"0499812\", \"0348887\", \"0771415\", \"1096068\", \"0664659\", \"0700686\", \"0675028\", \"1173520\"]\n",
      "batch[\"__key__\"] = [\"1185829\", \"0850691\", \"0750571\", \"1018058\", \"0657336\", \"0861467\", \"1043187\", \"1107649\", \"0091504\", \"0000983\", \"0737375\", \"0485732\", \"1236601\", \"0764397\", \"0790870\", \"0041730\", \"0190237\", \"1148516\", \"0922603\", \"0819831\", \"0829031\", \"1264578\", \"0352931\", \"1160970\", \"0495205\", \"0595037\", \"0016206\", \"0844710\", \"0969325\", \"0424094\", \"1169054\", \"0765934\"]\n",
      "batch[\"__key__\"] = [\"1259573\", \"0845796\", \"0253960\", \"0899861\", \"1081539\", \"1252956\", \"0637981\", \"0856530\", \"0326032\", \"0009573\", \"0262902\", \"0680924\", \"0331474\", \"1086440\", \"0891945\", \"0565787\", \"0582310\", \"0697840\", \"0432694\", \"0525975\", \"0031798\", \"0611943\", \"0945603\", \"1146219\", \"0233596\", \"1018356\", \"1118025\", \"0887618\", \"0819627\", \"0548491\", \"0123284\", \"0241550\"]\n",
      "batch[\"__key__\"] = [\"0002301\", \"0909388\", \"0552143\", \"0524829\", \"0926439\", \"0762509\", \"0676577\", \"0673152\", \"0269782\", \"0767306\", \"0856287\", \"0615068\", \"0366201\", \"0011403\", \"0840315\", \"0118045\", \"1062909\", \"0598777\", \"0267797\", \"0828043\", \"0752879\", \"1015752\", \"0054058\", \"0297076\", \"0995304\", \"0827894\", \"1249957\", \"1259913\", \"1104566\", \"0589421\", \"0380789\", \"0622713\"]\n",
      "batch[\"__key__\"] = [\"0516501\", \"0512357\", \"0656048\", \"1257176\", \"0648342\", \"0788879\", \"0889390\", \"0406719\", \"0990538\", \"0271785\", \"0656829\", \"0623471\", \"0517140\", \"0501235\", \"0451449\", \"1102443\", \"0896901\", \"1149667\", \"0791320\", \"0475981\", \"0384674\", \"1268318\", \"0550101\", \"0780531\", \"1252572\", \"1255185\", \"0867529\", \"1027693\", \"0647839\", \"1158401\", \"0336222\", \"0822510\"]\n",
      "batch[\"__key__\"] = [\"0447333\", \"0450085\", \"0294949\", \"1075511\", \"1036593\", \"0021862\", \"0128894\", \"0184526\", \"1262321\", \"0161569\", \"1164348\", \"1197217\", \"0334507\", \"0235139\", \"1177804\", \"0507002\", \"0458864\", \"0933994\", \"0022197\", \"0741824\", \"1025332\", \"1006163\", \"0030619\", \"0061110\", \"0562462\", \"0376335\", \"0490552\", \"0293310\", \"0460758\", \"0068495\", \"0478973\", \"0910273\"]\n",
      "batch[\"__key__\"] = [\"0620905\", \"0108040\", \"1079309\", \"1085863\", \"0494892\", \"0289143\", \"0156091\", \"0524195\", \"0297822\", \"0372540\", \"0199318\", \"0315182\", \"1081857\", \"0080700\", \"1053974\", \"0131077\", \"1123510\", \"0831736\", \"0998210\", \"1071592\", \"1132746\", \"0051708\", \"0563122\", \"0644032\", \"0202931\", \"1180375\", \"0833614\", \"1274191\", \"1129844\", \"0011027\", \"0625060\", \"0726180\"]\n",
      "batch[\"__key__\"] = [\"1045257\", \"0852478\", \"0619943\", \"0662929\", \"0106953\", \"0353119\", \"0935998\", \"0411307\", \"1082606\", \"1204500\", \"0496678\", \"0262435\", \"0732705\", \"1073676\", \"0268693\", \"1057685\", \"0033320\", \"0061035\", \"0901509\", \"0652968\", \"0361398\", \"0299739\", \"0597734\", \"0929624\", \"0396195\", \"0557757\", \"0133958\", \"0969722\", \"1037454\", \"0693778\", \"1261068\", \"0075111\"]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Channel{Any}(11) (11 items available)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "debugloader(desc, [\"test.tar\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0c45e024-4302-499f-8c84-fbc5325be2d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(count, total) = (1, 32)\n",
      "(count, total) = (101, 3232)\n",
      "(count, total) = (201, 6432)\n",
      "(count, total) = (301, 9632)\n",
      "(count, total) = (401, 12832)\n",
      "(count, total) = (501, 16032)\n",
      "(count, total) = (601, 19232)\n",
      "(count, total) = (701, 22432)\n",
      "(count, total) = (801, 25611)\n",
      "(count, total) = (901, 28811)\n",
      "(count, total) = (1001, 32011)\n",
      " 83.441163 seconds (12.55 M allocations: 70.612 GiB, 1.26% gc time, 0.04% compilation time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1064, 33964)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "desc.debug = false\n",
    "(count, total) = (0, 0)\n",
    "global last_sample\n",
    "@time for sample in dataloader(desc)\n",
    "    last_sample = sample\n",
    "    count += 1\n",
    "    total += length(sample[\"__key__\"])\n",
    "    if count % 100 == 1; @show count, total; end\n",
    "end\n",
    "(count, total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9feaae85-c9fc-4d63-bbe7-ad43e96e35ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "545.1685393258427"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "33964 / 62.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "39ad0630-a976-4e41-8ab6-7fe62c718c4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dict{Any, Any} with 3 entries:\n",
       "  \"__key__\" => [\"0132804\", \"0645902\", \"0892116\", \"0409602\", \"0959618\", \"1132342\", \"1273685\", \"1040510\", \"0…\n",
       "  \".jpg\"    => Matrix{T} where T[ColorTypes.RGB{FixedPointNumbers.N0f8}[RGB{N0f8}(0.325,0.275,0.098) RGB{N…\n",
       "  \".cls\"    => [102, 502, 695, 319, 749, 884, 994, 812, 32, 2, 707]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5b19ad9d-6c16-4178-9d7e-1a20ca3681cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "#5 (generic function with 1 method)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(x,y)->x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "976c7707-441b-4e60-b0f5-2785a138155a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "f (generic function with 1 method)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function f(x::Int)::Int\n",
    "    return x+1\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7546df73-5f77-4926-9829-a64663e573e2",
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "MethodError: no method matching f(::Int64, ::Int64)\n\u001b[0mClosest candidates are:\n\u001b[0m  f(::Int64) at In[8]:1",
     "output_type": "error",
     "traceback": [
      "MethodError: no method matching f(::Int64, ::Int64)\n\u001b[0mClosest candidates are:\n\u001b[0m  f(::Int64) at In[8]:1",
      "",
      "Stacktrace:",
      " [1] top-level scope",
      "   @ In[9]:1",
      " [2] eval",
      "   @ ./boot.jl:360 [inlined]",
      " [3] include_string(mapexpr::typeof(REPL.softscope), mod::Module, code::String, filename::String)",
      "   @ Base ./loading.jl:1094"
     ]
    }
   ],
   "source": [
    "f(3,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a09da1ac-a21d-41d0-b8c2-6d4965077382",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.6.0",
   "language": "julia",
   "name": "julia-1.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}