{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9ae7d6dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34fbefe6",
   "metadata": {},
   "source": [
    "## Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8f6f393e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['arr_0', 'arr_1']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.load('train_mnist.npz')\n",
    "data.files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "717e9c49",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainX = data['arr_0']\n",
    "trainy = data['arr_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ba71af57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x13cf33580>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAN8klEQVR4nO3df6jVdZ7H8ddrbfojxzI39iZOrWOEUdE6i9nSyjYRTj8o7FYMIzQ0JDl/JDSwyIb7xxSLIVu6rBSDDtXYMus0UJHFMNVm5S6BdDMrs21qoxjlphtmmv1a9b1/3K9xp+75nOs53/PD+34+4HDO+b7P93zffPHl99f53o8jQgAmvj/rdQMAuoOwA0kQdiAJwg4kQdiBJE7o5sJsc+of6LCI8FjT29qy277C9lu237F9ezvfBaCz3Op1dtuTJP1B0gJJOyW9JGlRROwozMOWHeiwTmzZ50l6JyLejYgvJf1G0sI2vg9AB7UT9hmS/jjq/c5q2p+wvcT2kO2hNpYFoE0dP0EXEeskrZPYjQd6qZ0t+y5JZ4x6/51qGoA+1E7YX5J0tu3v2j5R0o8kbaynLQB1a3k3PiIO2V4q6SlJkyQ9EBFv1NYZgFq1fOmtpYVxzA50XEd+VAPg+EHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEi0P2Yzjw6RJk4r1U045paPLX7p0acPaSSedVJx39uzZxfqtt95arN9zzz0Na4sWLSrO+/nnnxfrK1euLNbvvPPOYr0X2gq77fckHZB0WNKhiJhbR1MA6lfHlv3SiPiwhu8B0EEcswNJtBv2kPS07ZdtLxnrA7aX2B6yPdTmsgC0od3d+PkRscv2X0h6xvZ/R8Tm0R+IiHWS1kmS7WhzeQBa1NaWPSJ2Vc97JD0maV4dTQGoX8thtz3Z9pSjryX9QNL2uhoDUK92duMHJD1m++j3/HtE/L6WriaYM888s1g/8cQTi/WLL764WJ8/f37D2tSpU4vzXn/99cV6L+3cubNYX7NmTbE+ODjYsHbgwIHivK+++mqx/sILLxTr/ajlsEfEu5L+qsZeAHQQl96AJAg7kARhB5Ig7EAShB1IwhHd+1HbRP0F3Zw5c4r1TZs2Feudvs20Xx05cqRYv/nmm4v1Tz75pOVlDw8PF+sfffRRsf7WW2+1vOxOiwiPNZ0tO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kwXX2GkybNq1Y37JlS7E+a9asOtupVbPe9+3bV6xfeumlDWtffvllcd6svz9oF9fZgeQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJhmyuwd69e4v1ZcuWFetXX311sf7KK68U683+pHLJtm3bivUFCxYU6wcPHizWzzvvvIa12267rTgv6sWWHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeS4H72PnDyyScX682GF167dm3D2uLFi4vz3njjjcX6hg0binX0n5bvZ7f9gO09trePmjbN9jO2366eT62zWQD1G89u/K8kXfG1abdLejYizpb0bPUeQB9rGvaI2Czp678HXShpffV6vaRr620LQN1a/W38QEQcHSzrA0kDjT5oe4mkJS0uB0BN2r4RJiKidOItItZJWidxgg7opVYvve22PV2Squc99bUEoBNaDftGSTdVr2+S9Hg97QDolKa78bY3SPq+pNNs75T0c0krJf3W9mJJ70v6YSebnOj279/f1vwff/xxy/PecsstxfrDDz9crDcbYx39o2nYI2JRg9JlNfcCoIP4uSyQBGEHkiDsQBKEHUiCsANJcIvrBDB58uSGtSeeeKI47yWXXFKsX3nllcX6008/Xayj+xiyGUiOsANJEHYgCcIOJEHYgSQIO5AEYQeS4Dr7BHfWWWcV61u3bi3W9+3bV6w/99xzxfrQ0FDD2n333Vect5v/NicSrrMDyRF2IAnCDiRB2IEkCDuQBGEHkiDsQBJcZ09ucHCwWH/wwQeL9SlTprS87OXLlxfrDz30ULE+PDxcrGfFdXYgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSILr7Cg6//zzi/XVq1cX65dd1vpgv2vXri3WV6xYUazv2rWr5WUfz1q+zm77Adt7bG8fNe0O27tsb6seV9XZLID6jWc3/leSrhhj+r9ExJzq8bt62wJQt6Zhj4jNkvZ2oRcAHdTOCbqltl+rdvNPbfQh20tsD9lu/MfIAHRcq2H/haSzJM2RNCxpVaMPRsS6iJgbEXNbXBaAGrQU9ojYHRGHI+KIpF9KmldvWwDq1lLYbU8f9XZQ0vZGnwXQH5peZ7e9QdL3JZ0mabekn1fv50gKSe9J+mlENL25mOvsE8/UqVOL9WuuuaZhrdm98vaYl4u/smnTpmJ9wYIFxfpE1eg6+wnjmHHRGJPvb7sjAF3Fz2WBJAg7kARhB5Ig7EAShB1Igltc0TNffPFFsX7CCeWLRYcOHSrWL7/88oa1559/vjjv8Yw/JQ0kR9iBJAg7kARhB5Ig7EAShB1IgrADSTS96w25XXDBBcX6DTfcUKxfeOGFDWvNrqM3s2PHjmJ98+bNbX3/RMOWHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeS4Dr7BDd79uxifenSpcX6ddddV6yffvrpx9zTeB0+fLhYHx4u//XyI0eO1NnOcY8tO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kwXX240Cza9mLFo010O6IZtfRZ86c2UpLtRgaGirWV6xYUaxv3LixznYmvKZbdttn2H7O9g7bb9i+rZo+zfYztt+unk/tfLsAWjWe3fhDkv4+Is6V9DeSbrV9rqTbJT0bEWdLerZ6D6BPNQ17RAxHxNbq9QFJb0qaIWmhpPXVx9ZLurZDPQKowTEds9ueKel7krZIGoiIoz9O/kDSQIN5lkha0kaPAGow7rPxtr8t6RFJP4uI/aNrMTI65JiDNkbEuoiYGxFz2+oUQFvGFXbb39JI0H8dEY9Wk3fbnl7Vp0va05kWAdSh6W68bUu6X9KbEbF6VGmjpJskrayeH+9IhxPAwMCYRzhfOffcc4v1e++9t1g/55xzjrmnumzZsqVYv/vuuxvWHn+8/E+GW1TrNZ5j9r+V9GNJr9veVk1brpGQ/9b2YknvS/phRzoEUIumYY+I/5I05uDuki6rtx0AncLPZYEkCDuQBGEHkiDsQBKEHUiCW1zHadq0aQ1ra9euLc47Z86cYn3WrFmttFSLF198sVhftWpVsf7UU08V65999tkx94TOYMsOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0mkuc5+0UUXFevLli0r1ufNm9ewNmPGjJZ6qsunn37asLZmzZrivHfddVexfvDgwZZ6Qv9hyw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSaS5zj44ONhWvR07duwo1p988sli/dChQ8V66Z7zffv2FedFHmzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJR0T5A/YZkh6SNCApJK2LiH+1fYekWyT9b/XR5RHxuybfVV4YgLZFxJijLo8n7NMlTY+IrbanSHpZ0rUaGY/9k4i4Z7xNEHag8xqFfTzjsw9LGq5eH7D9pqTe/mkWAMfsmI7Zbc+U9D1JW6pJS22/ZvsB26c2mGeJ7SHbQ+21CqAdTXfjv/qg/W1JL0haERGP2h6Q9KFGjuP/SSO7+jc3+Q5244EOa/mYXZJsf0vSk5KeiojVY9RnSnoyIs5v8j2EHeiwRmFvuhtv25Lul/Tm6KBXJ+6OGpS0vd0mAXTOeM7Gz5f0n5Jel3Skmrxc0iJJczSyG/+epJ9WJ/NK38WWHeiwtnbj60LYgc5reTcewMRA2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSKLbQzZ/KOn9Ue9Pq6b1o37trV/7kuitVXX29peNCl29n/0bC7eHImJuzxoo6Nfe+rUvid5a1a3e2I0HkiDsQBK9Dvu6Hi+/pF9769e+JHprVVd66+kxO4Du6fWWHUCXEHYgiZ6E3fYVtt+y/Y7t23vRQyO237P9uu1tvR6frhpDb4/t7aOmTbP9jO23q+cxx9jrUW932N5Vrbtttq/qUW9n2H7O9g7bb9i+rZre03VX6Ksr663rx+y2J0n6g6QFknZKeknSoojY0dVGGrD9nqS5EdHzH2DY/jtJn0h66OjQWrb/WdLeiFhZ/Ud5akT8Q5/0doeOcRjvDvXWaJjxn6iH667O4c9b0Yst+zxJ70TEuxHxpaTfSFrYgz76XkRslrT3a5MXSlpfvV6vkX8sXdegt74QEcMRsbV6fUDS0WHGe7ruCn11RS/CPkPSH0e936n+Gu89JD1t+2XbS3rdzBgGRg2z9YGkgV42M4amw3h309eGGe+bddfK8Oft4gTdN82PiL+WdKWkW6vd1b4UI8dg/XTt9BeSztLIGIDDklb1splqmPFHJP0sIvaPrvVy3Y3RV1fWWy/CvkvSGaPef6ea1hciYlf1vEfSYxo57Ognu4+OoFs97+lxP1+JiN0RcTgijkj6pXq47qphxh+R9OuIeLSa3PN1N1Zf3VpvvQj7S5LOtv1d2ydK+pGkjT3o4xtsT65OnMj2ZEk/UP8NRb1R0k3V65skPd7DXv5Evwzj3WiYcfV43fV8+POI6PpD0lUaOSP/P5L+sRc9NOhrlqRXq8cbve5N0gaN7Nb9n0bObSyW9OeSnpX0tqT/kDStj3r7N40M7f2aRoI1vUe9zdfILvprkrZVj6t6ve4KfXVlvfFzWSAJTtABSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBL/DyJ7caZa7LphAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(trainX[0], cmap=plt.get_cmap('gray'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "201fb4e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAyAAAAGdCAYAAADquxfqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA650lEQVR4nO3df/zV8/3/8ftDKpEi+bQW/Rj50YyaGNYHnynDosZQI2UmH8big9XMxobJhs/8mNFIGRfmM1Gz+aiRXxs+YXZZKiqfRaQykt9Jz+8fzvbt8fzUOe/zPuf1PK9zzu36T+/7eb/f5/WYnbvT0+v1PC8LIQgAAAAAUtik1gMAAAAAaB4sQAAAAAAkwwIEAAAAQDIsQAAAAAAkwwIEAAAAQDIsQAAAAAAkU9ECxMwOMbPnzWyRmU2o1lAAPLoGZI+eAdmjZ5Aka+19QMysjaQXJA2RtFTSHEkjQwjzivwONx2pb6+HELat9RDNptyu0bO6R89qgPe05hNCsFrP0GzoWVPa4HtaJWdA9pa0KITwYghhjaQ7JA2r4PmQf0tqPUCTomvNhZ7VBj0DskfPms8G39MqWYD0kPTyenlp4THHzMaa2VNm9lQFxwKaWcmu0TOgYrynAdmjZ5AkbZr1AUIIkyRNkjiNBmSFngFp0DUge/Ss8VVyBuQVSduvl7crPAaguugakD16BmSPnkFSZQuQOZL6mlkfM2snaYSkGdUZC8B66BqQPXoGZI+eQVIFl2CFENaa2emS7pfURtLkEMJzVZsMgCS6BqRAz4Ds0TP8Q6s/hrdVB+M6vnr3dAhhYK2HQHH0rO7RszpB1+obH8NbH+hZ3dvgexp3QgcAAACQDAsQAAAAAMmwAAEAAACQDAsQAAAAAMmwAAEAAACQDAsQAAAAAMmwAAEAAACQDAsQAAAAAMm0+k7oAJAHe+65p8unn366yyeccILLt9xyi8vXXHONy88880wVpwMAADHOgAAAAABIhgUIAAAAgGRYgAAAAABIxkII6Q5mlu5gGWjTpo3LnTt3Luv342vTN998c5d33nlnl7/1rW+5fPnll7s8cuRIlz/44AOXJ06c6PIPf/jDlg+7YU+HEAZW+iTIVr33rJT+/fu7/OCDD7rcqVOnsp7vrbfecnmbbbZp1VxVRM/qRKN3LWsHHXSQy7fddpvLBxxwgMvPP/98VY8fQrCqPiEyQc+KO//8812O/663ySb+XMOBBx7o8sMPP5zJXOvZ4HsaZ0AAAAAAJMMCBAAAAEAyLEAAAAAAJNNU9wHp2bOny+3atXN5v/32c3nQoEEub7XVVi4fddRR1RtO0tKlS12++uqrXf7qV7/q8ttvv+3yX/7yF5cTXNcHZG7vvfd2+a677nI53osV72uLe7JmzRqX4z0f++yzj8vxfUHi3wdaa//993c5fi3efffdKcdJbq+99nJ5zpw5NZoEqB9jxoxxefz48S6vW7eu6O+n3PtdDGdAAAAAACTDAgQAAABAMixAAAAAACTT0HtASt0voNz7eFRbfJ1e/FnO77zzjsvxZ6QvW7bM5TfffNPlan9mOpCF+H44n//8512+9dZbXe7evXtZz79w4UKXf/KTn7h8xx13uPzHP/7R5biXl156aVnHBzYm/jz+vn37utxoe0Di+xH06dPH5V69erlsxm06gFjck80226xGk1SGMyAAAAAAkmEBAgAAACAZFiAAAAAAkmnoPSAvvfSSy3//+99drvYekCeffNLlVatWufxv//ZvLsf3E/jVr35V1XmAenDDDTe4PHLkyKo+f7ynpGPHji7H98uJr8vffffdqzoP8A8nnHCCy48//niNJkkj3r918sknuxzv91qwYEHmMwF5N3jwYJfPOOOMoj8f92bo0KEuL1++vDqDVYgzIAAAAACSYQECAAAAIBkWIAAAAACSaeg9IG+88YbL5557rsvxdXF//vOfXb766quLPv+zzz7r8pAhQ1x+9913Xf7sZz/r8rhx44o+P9CI9txzT5e/8pWvuFzqs//jPRu//e1vXb788stdfvXVV12Oex7fP+dLX/pSWfMArRXfF6PR3XjjjUW/H9+zB2hGgwYNcvnmm292udT+5Z/+9KcuL1mypDqDVVlz/dsPAAAAQE2xAAEAAACQTMkFiJlNNrMVZjZ3vce6mNksM1tY+HPrbMcEGh9dA7JHz4Ds0TOUYiGE4j9gtr+kdyTdEkLYrfDYTyS9EUKYaGYTJG0dQhhf8mBmxQ+WWKdOnVx+++23XY7vT3DSSSe5fPzxx7t8++23V3G6XHo6hDCw1kM0qmp1LW8969+/v8sPPvigy3EPY/fdd5/L8X1CDjjgAJfj+3bE152vXLmy6PE+/vhjl997772ix3vmmWeKPl8r0LMM1fI9LX5txvf9mDZtmsujRo0q5+lz709/+pPL++yzj8v77befy0888USm84QQ2OCVkUb+u2PWfvnLX7r8jW98o+jPP/TQQy4fdNBB1R6pUht8Tyt5BiSE8IikN6KHh0maWvh6qqThlU4HNDu6BmSPngHZo2copbWfgtUthLCs8PVrkrpt7AfNbKyksa08DtDsWtQ1egZUhPc0IHv0DP9U8cfwhhBCsdNjIYRJkiZJzXcaDaimYl2jZ0B18J4GZI+eobULkOVm1j2EsMzMuktaUc2hUlm9enXR77/11ltFv3/yySe7/Otf/9rldevWtW4w4P+ru67ttNNOLsf334k/w/z11193edmyZS5PnTrV5Xfeecfl3/3ud0VzpTp06ODy2Wef7fJxxx1X1eOhJpL07LDDDnM5fm01mm7d/H/g7tOnT9Gff+WVV7IcB7VXd+9nKXTt2tXleM9H/HfJVatWuXzxxRdnMlfWWvsxvDMkjS58PVrS9OqMAyBC14Ds0TMge/QM/9SSj+G9XdLjknY2s6VmdpKkiZKGmNlCSYMLGUAF6BqQPXoGZI+eoZSSl2CFEEZu5Fu5+5wvoJ7RNSB79AzIHj1DKRVvQm9kF154oct77rmny/H9AAYPHuzyzJkzM5kLyJP27du7fPnll7scX/ce32/nhBNOcPmpp55yOW/Xyffs2bPWI6BO7bzzzkW//9xzzyWaJI343wXxnpAXXnjB5fjfDUAj6t27t8t33XVXWb9/zTXXuDx79uxKR6qJ1u4BAQAAAICysQABAAAAkAwLEAAAAADJsAekiHfffdfl+L4fzzzzjMu//OUvXY6vy4uvbf/5z3/ucgjcawf1Z8CAAS7Hez5iw4YNc/nhhx+u+kxAPZozZ06tRyiqU6dOLh9yyCEuH3/88S4ffPDBRZ/voosucjm+vwHQiOLe7L777kV//oEHHnD5qquuqvpMtcAZEAAAAADJsAABAAAAkAwLEAAAAADJsAekDIsXL3Z5zJgxLt98880ujxo1qmjeYostXL7llltcXrZsWWvGBJK68sorXTYzl+M9Hnnf87HJJv6/y6xbt65Gk6DZdOnSpaLf32OPPVyOuxjfq2q77bZzuV27di4fd9xxLsfdeP/9911+8sknXf7www9d3nRT/1eOp59+WkCjGz58uMsTJxa/Afxjjz3m8ujRo11+6623qjJXrXEGBAAAAEAyLEAAAAAAJMMCBAAAAEAy7AGpwN133+3ywoULXY6vjT/ooINc/vGPf+xyr169XL7kkktcfuWVV1o1J1BNQ4cOdbl///4ux/ezmTFjRtYjVVW85yP+3/Pss88mnAaNJN4zEb+2rr/+epfPO++8sp4/vp9AvAdk7dq1Lr/33nsuz5s3z+XJkye7HN/LKt7PtXz5cpeXLl3qcocOHVxesGCBgEbTu3dvl++6666yfv/FF190Oe5Vo+AMCAAAAIBkWIAAAAAASIYFCAAAAIBk2ANSRXPnznX5mGOOcfnwww93Ob5vyCmnnOJy3759XR4yZEilIwIVi6/jju8dsGLFCpd//etfZz5TOdq3b+/yhRdeWPTnH3zwQZe/+93vVnskNInTTjvN5SVLlri83377VfT8L730ksv33HOPy/Pnz3f5iSeeqOh4sbFjx7q87bbbuhxf2w40ovHjx7tc7r2kSt0npFFwBgQAAABAMixAAAAAACTDAgQAAABAMuwBydCqVatc/tWvfuXyjTfe6PKmm/r/O/bff3+XDzzwQJcfeuihiuYDsvDhhx+6vGzZshpN8ol4z8f555/v8rnnnutyfO+CK664wuV33nmnitOhmV122WW1HqGq4ntdxcq9HwJQD+J7YR188MFl/f706dNdfv755ysdqS5wBgQAAABAMixAAAAAACTDAgQAAABAMuwBqaLdd9/d5a997Wsu77XXXi7Hez5i8+bNc/mRRx6pYDogjRkzZtT0+PH1uPEej2OPPdbl+Prbo446KpO5gGZ3991313oEoOpmzpzp8tZbb1305+P774wZM6baI9UFzoAAAAAASIYFCAAAAIBkWIAAAAAASIY9IGXYeeedXT799NNdPvLII13+1Kc+Vdbzf/zxxy7H909Yt25dWc8HZMHMiubhw4e7PG7cuEznOeuss1z+/ve/73Lnzp1dvu2221w+4YQTshkMANDwttlmG5dL/V3tuuuuc7lZ7y3FGRAAAAAAybAAAQAAAJBMyQWImW1vZrPNbJ6ZPWdm4wqPdzGzWWa2sPBn8c8dA7BR9AxIg64B2aNnKKUle0DWSjo7hPCMmW0p6WkzmyVpjKQHQggTzWyCpAmSxmc3avbiPRsjR450Od7z0bt374qO99RTT7l8ySWXuFzr+ykgqbrpWQihaI57dPXVV7s8efJkl//+97+7vM8++7g8atQol/fYYw+Xt9tuO5dfeukll++//36X4+tv0XTqpmv1Lt4fttNOO7kc3w8BDaVhe3bzzTe7vMkm5V1M9Kc//ama49Stkv/UQgjLQgjPFL5+W9J8ST0kDZM0tfBjUyUNz2hGoOHRMyANugZkj56hlLI+BcvMeksaIOlJSd1CCP/4mKbXJHXbyO+MlTS2ghmBpkLPgDToGpA9eoYNafF5IzPrKOkuSWeGEFav/73wyTUYYUO/F0KYFEIYGEIYWNGkQBOgZ0AadA3IHj3DxrToDIiZtdUnL6DbQgjTCg8vN7PuIYRlZtZd0oqshqyWbt38Qrtfv34uX3vttS7vsssuFR3vySefdPmnP/2py9OnT3eZ+3w0t0bpWZs2bVw+7bTTXD7qqKNcXr3avSepb9++ZR0vvp529uzZLv/gBz8o6/nQ+Bqla3kX7w8r91p51LdG6Vn//v1dHjx4sMvx393WrFnj8s9//nOXly9fXr3h6lhLPgXLJN0kaX4I4cr1vjVD0ujC16MlTY9/F0DL0DMgDboGZI+eoZSWnAH5oqRRkv5qZs8WHjtP0kRJd5rZSZKWSDomkwmB5kDPgDToGpA9eoaiSi5AQgiPSbKNfPug6o4DNCd6BqRB14Ds0TOUUtanYOVdly5dXL7hhhtcjq/j+8xnPlPR8eJrz6+44gqX4/sPvP/++xUdD8iDxx9/3OU5c+a4vNdeexX9/fg+IfHerFh8n5A77rjD5XHjxhX9fQD5sO+++7o8ZcqU2gwClGGrrbZyOX4Pi73yyisun3POOdUeqSGwIwwAAABAMixAAAAAACTDAgQAAABAMnW1B+QLX/iCy+eee67Le++9t8s9evSo6Hjvvfeey1dffbXLP/7xj11+9913KzoeUA+WLl3q8pFHHunyKaec4vL5559f1vNfddVVLv/iF79wedGiRWU9H4Da+OSTWAHg/+IMCAAAAIBkWIAAAAAASIYFCAAAAIBk6moPyFe/+tWiuZR58+a5fO+997q8du1al+P7eqxataqs4wHNYNmyZS5feOGFRTOAxnTfffe5fPTRR9doEqB6FixY4HJ8D7hBgwalHKdhcAYEAAAAQDIsQAAAAAAkwwIEAAAAQDIWQkh3MLN0B0MWng4hDKz1ECiOntU9elYn6Fp9CyFwo5I6QM/q3gbf0zgDAgAAACAZFiAAAAAAkmEBAgAAACAZFiAAAAAAkmEBAgAAACAZFiAAAAAAkmEBAgAAACAZFiAAAAAAkmEBAgAAACAZFiAAAAAAkmEBAgAAACCZTRMf73VJSyR1LXydV8y3Yb1qcEyUj55VBz1DKfXQtTzPJtEzlFYPPZOYb2M22DULIaQeRGb2VAhhYPIDtxDzoRHk/XXCfGgUeX6t5Hk2Kf/zIT/y/lphvvJwCRYAAACAZFiAAAAAAEimVguQSTU6bksxHxpB3l8nzIdGkefXSp5nk/I/H/Ij768V5itDTfaAAAAAAGhOXIIFAAAAIBkWIAAAAACSSboAMbNDzOx5M1tkZhNSHntjzGyyma0ws7nrPdbFzGaZ2cLCn1vXaLbtzWy2mc0zs+fMbFye5kN+5a1ree5ZYRa6hrLRs7Lno2coW956JuW7a/XSs2QLEDNrI+nnkg6V1E/SSDPrl+r4RUyRdEj02ARJD4QQ+kp6oJBrYa2ks0MI/STtI+lbhX9meZkPOZTTrk1Rfnsm0TWUiZ61Cj1DWXLaMynfXauLnqU8A7K3pEUhhBdDCGsk3SFpWMLjb1AI4RFJb0QPD5M0tfD1VEnDU870DyGEZSGEZwpfvy1pvqQeeZkPuZW7ruW5ZxJdQ6vQszLRM7RC7nom5btr9dKzlAuQHpJeXi8vLTyWR91CCMsKX78mqVsth5EkM+staYCkJ5XD+ZAr9dK1XL6O6RpaiJ5VgJ6hheqlZ1IOX8d57hmb0EsIn3xOcU0/q9jMOkq6S9KZIYTV638vD/MBlcrL65iuoZHl5TVMz9Do8vA6znvPUi5AXpG0/Xp5u8JjebTczLpLUuHPFbUaxMza6pMX0G0hhGl5mw+5VC9dy9XrmK6hTPSsFegZylQvPZNy9Dquh56lXIDMkdTXzPqYWTtJIyTNSHj8csyQNLrw9WhJ02sxhJmZpJskzQ8hXLnet3IxH3KrXrqWm9cxXUMr0LMy0TO0Qr30TMrJ67heepb0Tuhmdpikn0lqI2lyCOGSZAffCDO7XdKBkrpKWi7pAkn3SLpTUk9JSyQdE0KINxulmG2QpEcl/VXSusLD5+mTa/lqPh/yK29dy3PPCvPRNZSNnpU9Hz1D2fLWMynfXauXniVdgAAAAABobmxCBwAAAJAMCxAAAAAAybAAAQAAAJAMCxAAAAAAybAAAQAAAJBMRQsQMzvEzJ43s0VmNqFaQwHw6BqQPXoGZI+eQargY3jNrI2kFyQNkbRUn9wsZmQIYV6R3+Ezf+vb6yGEbWs9RLMpt2v0rO7RsxrgPa35hBCs1jM0G3rWlDb4nlbJGZC9JS0KIbwYQlgj6Q5Jwyp4PuTfkloP0KToWnOhZ7VBz4Ds0bPms8H3tEoWID0kvbxeXlp4zDGzsWb2lJk9VcGxgGZWsmv0DKgY72lA9ugZJEmbZn2AEMIkSZMkTqMBWaFnQBp0DcgePWt8lZwBeUXS9uvl7QqPAaguugZkj54B2aNnkFTZAmSOpL5m1sfM2kkaIWlGdcYCsB66BmSPngHZo2eQVMElWCGEtWZ2uqT7JbWRNDmE8FzVJgMgia4BKdAzIHv0DP/Q6o/hbdXBuI6v3j0dQhhY6yFQHD2re/SsTtC1+sbH8NYHelb3Nviexp3QAQAAACTDAgQAAABAMixAAAAAACTDAgQAAABAMixAAAAAACTDAgQAAABAMixAAAAAACTT6hsRAgCA+nXVVVe5/O1vf9vluXPnujx06FCXlyxZks1gABoeZ0AAAAAAJMMCBAAAAEAyLEAAAAAAJMMeEAB1bcstt3S5Y8eOLn/lK19xedttt3X5yiuvdPnDDz+s4nRAfvTu3dvl448/3uV169a5vOuuu7q8yy67uMweEOD/2mmnnVxu27aty/vvv7/L1113nctxDys1ffp0l0eMGOHymjVrqnq8luIMCAAAAIBkWIAAAAAASIYFCAAAAIBk2AMCINfi69bHjx/v8r777uvybrvtVtbzd+/e3eX4XghAo1i5cqXLjzzyiMtHHHFEynGAuvTZz37W5TFjxrh89NFHu7zJJv6/9X/60592Od7zEUKocEIv7vX111/v8plnnuny6tWrq3r8jeEMCAAAAIBkWIAAAAAASIYFCAAAAIBk2ANShi984Qsux5+hfsABB7gcXycYO+ecc1x+9dVXXR40aJDLt956q8tPPvlk0ecH6kF8b4H4etTjjjvO5Q4dOrhsZi6//PLLLr/99tsux/c2OOaYY1yOP5N9wYIFG5gaqD/vvvuuy9zHAyjfpZde6vJhhx1Wo0la54QTTnD5pptucvmPf/xjkjk4AwIAAAAgGRYgAAAAAJJhAQIAAAAgGfaAFHHssce6fNVVV7nctWtXl+Nr0R966CGXt912W5d/+tOfFj1+/Hzx748YMaLo7wN50LlzZ5cvu+wyl+OebbnllmU9/8KFC13+8pe/7HLbtm1djvd0xD2OM9AottpqK5f32GOP2gwC1LFZs2a5XGoPyIoVK1yO91zE9wmJ7wsS22+//VyO9x/XC86AAAAAAEiGBQgAAACAZFiAAAAAAEimqfeAbLqp/58/cOBAl3/5y1+6vPnmm7v8yCOPuHzRRRe5/Nhjj7ncvn17l++8806XDz744KLzPvXUU0W/D+TRV7/6VZe/+c1vVvR8ixcvdnnIkCEux/cB2XHHHSs6HtAo4vewnj17lvX7e+21l8vxfiruK4Jm8Itf/MLle+65p+jPf/TRRy6/9tprFR2/U6dOLs+dO9flT3/600V/P563Vn+35AwIAAAAgGRYgAAAAABIpuQCxMwmm9kKM5u73mNdzGyWmS0s/Ll1tmMCjY+uAdmjZ0D26BlKackekCmSrpV0y3qPTZD0QAhhoplNKOTx1R8vW8cff7zLN954Y9Gfjz/7Ob5/werVq4v+fvzzpfZ8LF261OWpU6cW/XnUvSlqwK4dffTRZf383/72N5fnzJnj8vjx/n9+vOcjtuuuu5Z1fDS8KWrAnrXEq6++6vKUKVNcvvDCC4v+fvz9VatWuXzttde2cjI0oClq0J6tXbvW5VLvQdUW3+tq663LW8fFf7f88MMPK56pNUqeAQkhPCLpjejhYZL+8bfhqZKGV3csoPnQNSB79AzIHj1DKa3dA9IthLCs8PVrkrpVaR4AHl0DskfPgOzRM/xTxR/DG0IIZhY29n0zGytpbKXHAZpdsa7RM6A6eE8DskfP0NoFyHIz6x5CWGZm3SWt2NgPhhAmSZokScVebCnE9+k477zzXA7Bj3fddde5fP7557tcas9H7Hvf+15ZP//tb3/b5ZUrV5b1+2gILepannoWO/nkk10eO9a/p8ycOdPlRYsWubxixUb/9dIi3brxH9lQUl2+p1Uqfk8stQcEqFBT9qxSI0aMcDl+T+3QoUNZz/eDH/yg4pmqobWXYM2QNLrw9WhJ06szDoAIXQOyR8+A7NEz/FNLPob3dkmPS9rZzJaa2UmSJkoaYmYLJQ0uZAAVoGtA9ugZkD16hlJKXoIVQhi5kW8dVOVZgKZG14Ds0TMge/QMpVS8CT3P4uvc4j0fa9ascfn+++93Ob7fwPvvv1/0eJtttpnL8X0+evbs6bKZuXzxxRe7PH06ZydR/+J7D6S+znzfffdNejygXm2yib8oYt26dTWaBGgexx13nMsTJkxweccdd3S5bdu2ZT3/s88+6/JHH31U1u9npbV7QAAAAACgbCxAAAAAACTDAgQAAABAMg21B2SrrbZy+bTTTnM5vs9HvOdj+PDhZR0vvi7vtttuc3nPPfcs+vu/+c1vXP7JT35S1vGBZhDfD2eLLbYo6/c/97nPFf3+n/70J5cff/zxsp4faBTxno/4PROA1Lt3b5dHjRrl8uDBg8t6vkGDBrlcbu/ie9LFe0h+//vfu1xqP3MqnAEBAAAAkAwLEAAAAADJsAABAAAAkExD7QFp166dy127di368/G15f/yL//i8oknnujyEUcc4fJuu+3mcseOHV2Or+OL86233uryu+++W3ReoBFsvvnmLvfr18/lCy64wOXDDjus6POVe++C+L4kcc8//vjjor8PAGge8d/1ZsyY4XJ8j7fUHn30UZcnTZpUo0nKwxkQAAAAAMmwAAEAAACQDAsQAAAAAMk01B6QNWvWuLxy5UqXt912W5f/93//1+VyP3s5vpY8/izm7t27u/z666+7/Nvf/ras4wH1oG3bti4PGDDA5bvuusvluCfxZ5THPYvv03HIIYe4HO8xiW26qf/X3pFHHunyVVdd5XL87xUAQPMys6K5XOXuY4wNHTrU5UMPPdTl++67r3WDZYwzIAAAAACSYQECAAAAIBkWIAAAAACSaag9IKtWrXJ5+PDhLt97770ud+nSxeXFixe7PH36dJenTJni8htvvOHyHXfc4XJ8bXv8faARxPffifdkTJs2rejv//CHP3T5wQcfdPmPf/yjy3Fv45+PP7M9Fu8Fu/TSS11+6aWXXL7nnntc/vDDD4s+P1Cvyr0Wff/993f52muvrfpMQK3NnTvX5QMPPNDl448/3uX777/f5Q8++KCi45900kkun3HGGRU9X15wBgQAAABAMixAAAAAACTDAgQAAABAMlbuvS8qOphZuoMlEF//+vDDD7scXz975plnunzNNddkMleGng4hDKz1ECgu657F9/n40Y9+5PK5555b9PfjzyQfNWqUy/FernjPxu9//3uXP//5z7sc37fjJz/5icvxHpFhw4YVnfcPf/iDy5dddpnLb775ZtHff/bZZ4t+fwPoWZ1otPe0jz/+2OVy/36w++67uzxv3ryKZ8pSCKGyGzggiUbrWbk6d+7s8t///veiP3/44Ye7nIP7gGzwPY0zIAAAAACSYQECAAAAIBkWIAAAAACSaaj7gKTWoUMHl+M9H/H1s9wHBPWoTZs2Ll900UUun3POOS6/++67Lk+YMMHluAfxno+BA/2lovG9BQYMGODywoULXT711FNdnj17tsudOnVyeb/99nP5uOOOc/mII45wedasWSrm5ZdfdrlPnz5Ffx7Ii+uvv97lU045pazfHzt2rMvxvkcA5fvyl79c6xEywRkQAAAAAMmwAAEAAACQDAsQAAAAAMmwB6QC999/f61HADIXX9cd7/l47733XI6vG585c6bL++yzj8snnniiy4ceeqjL8V6r+L4jN998s8vxHozY6tWrXf7v//7vonnkyJEuf/3rXy/6/GeddVbR7wN5tWDBglqPACQX39vq4IMPdvnBBx90+f333890nvg98aqrrsr0eLXCGRAAAAAAybAAAQAAAJBMyQWImW1vZrPNbJ6ZPWdm4wqPdzGzWWa2sPDn1tmPCzQmegakQdeA7NEzlGLxvSr+zw+YdZfUPYTwjJltKelpScMljZH0RghhoplNkLR1CGF8iecqfrA6E3828+9//3uX43+23bt3d3nlypXZDJadp0MIA0v/GMqV554tW7bM5W233dblDz/80OX4OvItttjC5R133LGs41944YUuX3rppS5//PHHZT1fHaBnGcpz1/LmhRdecHmHHXYo+vObbOL/m2bc9cWLF1dnsCoJIVitZ2hUee7ZoEGDXP7e977n8pAhQ1yO7+VUap9hKV26dHH5sMMOc/maa65xecsttyz6fPGelPjeVfG9sGpgg+9pJc+AhBCWhRCeKXz9tqT5knpIGiZpauHHpuqTFxaAVqBnQBp0DcgePUMpZe0BMbPekgZIelJStxDCP/7T6GuSulV3NKA50TMgDboGZI+eYUNa/DG8ZtZR0l2SzgwhrDb7/2cuQwhhY6fIzGyspLEb+h4Aj54BadA1IHv0DBvTogWImbXVJy+g20II0woPLzez7iGEZYVr/VZs6HdDCJMkTSo8T0NdL/uZz3ym1iOggeS1Z6+99prL8R6Q9u3bu7zHHnsUfb54r9Qjjzzi8j333OPy3/72N5cbcM8HEstr1/Lmueeec7nUe966deuyHAd1Jq89u/baa13ebbfdiv78d77zHZfffvvtio4f7zH5/Oc/73KpvdkPPfSQy7/4xS9czsGejxZpyadgmaSbJM0PIVy53rdmSBpd+Hq0pOnVHw9oDvQMSIOuAdmjZyilJWdAvihplKS/mtmzhcfOkzRR0p1mdpKkJZKOyWRCoDnQMyANugZkj56hqJILkBDCY5I29lF1B1V3HKA50TMgDboGZI+eoZQWb0LH//Xoo4+6HH8GOtfDohHsv//+Lg8fPtzl+PrVFSv8Jb2TJ092+c0333R5zZo1FU4IIAuTJk1y+fDDD6/RJEDtnHrqqUmPF7+H/va3v3V53LhxLn/wwQeZz5SFsj6GFwAAAAAqwQIEAAAAQDIsQAAAAAAkY6U+b7iqB2vwz0x/4YUXXI4/M33QoEEuP/HEE5nPVGVPhxAG1noIFNfoPWsC9KxONHrXevXq5fK9997r8q677ury+jeZk6SddtrJ5cWLF1dxusqFEDa2SRo5Uu2e9e/f3+UzzjjD5dGjR6ua4tf9e++953K8nzjeezV37tyqzlMDG3xP4wwIAAAAgGRYgAAAAABIhgUIAAAAgGTYA1JFY8aMcfnGG290+eGHH3Y5vu5w3rx5mcxVRVybXgcavWdNgJ7VCbpW39gDUh+y7ln79u1djv8ud/HFF7u89dZbu3zPPfe4PGvWLJenT5/u8muvvdaKKesae0AAAAAA1BYLEAAAAADJsAABAAAAkAx7QKqoU6dOLt95550uDx482OVp06a5fOKJJ7r87rvvVnG6quDa9DrQ6D1rAvSsTtC1+sYekPpAz+oee0AAAAAA1BYLEAAAAADJsAABAAAAkAx7QDIU7wm55JJLXD711FNd3n333V3O4X1BuDa9DjRbzxoQPasTdK2+sQekPtCzusceEAAAAAC1xQIEAAAAQDIsQAAAAAAkwx4QlINr0+sAPat79KxO0LX6xh6Q+kDP6h57QAAAAADUFgsQAAAAAMmwAAEAAACQzKaJj/e6pCWSuha+zivm27BeNTgmykfPqoOeoZR66FqeZ5PoGUqrh55JzLcxG+xa0k3o/zyo2VN53mTJfGgEeX+dMB8aRZ5fK3meTcr/fMiPvL9WmK88XIIFAAAAIBkWIAAAAACSqdUCZFKNjttSzIdGkPfXCfOhUeT5tZLn2aT8z4f8yPtrhfnKUJM9IAAAAACaE5dgAQAAAEiGBQgAAACAZJIuQMzsEDN73swWmdmElMfeGDObbGYrzGzueo91MbNZZraw8OfWNZptezObbWbzzOw5MxuXp/mQX3nrWp57VpiFrqFs9Kzs+egZypa3nkn57lq99CzZAsTM2kj6uaRDJfWTNNLM+qU6fhFTJB0SPTZB0gMhhL6SHijkWlgr6ewQQj9J+0j6VuGfWV7mQw7ltGtTlN+eSXQNZaJnrULPUJac9kzKd9fqomcpz4DsLWlRCOHFEMIaSXdIGpbw+BsUQnhE0hvRw8MkTS18PVXS8JQz/UMIYVkI4ZnC129Lmi+pR17mQ27lrmt57plE19Aq9KxM9AytkLueSfnuWr30LOUCpIekl9fLSwuP5VG3EMKywtevSepWy2Ekycx6Sxog6UnlcD7kSr10LZevY7qGFqJnFaBnaKF66ZmUw9dxnnvGJvQSwiefU1zTzyo2s46S7pJ0Zghh9frfy8N8QKXy8jqma2hkeXkN0zM0ujy8jvPes5QLkFckbb9e3q7wWB4tN7PuklT4c0WtBjGztvrkBXRbCGFa3uZDLtVL13L1OqZrKBM9awV6hjLVS8+kHL2O66FnKRcgcyT1NbM+ZtZO0ghJMxIevxwzJI0ufD1a0vRaDGFmJukmSfNDCFeu961czIfcqpeu5eZ1TNfQCvSsTPQMrVAvPZNy8jqul54lvRO6mR0m6WeS2kiaHEK4JNnBN8LMbpd0oKSukpZLukDSPZLulNRT0hJJx4QQ4s1GKWYbJOlRSX+VtK7w8Hn65Fq+ms+H/Mpb1/Lcs8J8dA1lo2dlz0fPULa89UzKd9fqpWdJFyAAAAAAmhub0AEAAAAkwwIEAAAAQDIsQAAAAAAkwwIEAAAAQDIsQAAAAAAkU9ECxMwOMbPnzWyRmU2o1lAAPLoGZI+eAdmjZ5Aq+BheM2sj6QVJQyQt1Sc3ixkZQphX5Hf4zN/69noIYdtaD9Fsyu0aPat79KwGeE9rPiEEq/UMzYaeNaUNvqdVcgZkb0mLQggvhhDWSLpD0rAKng/5t6TWAzQputZc6Flt0DMge/Ss+WzwPa2SBUgPSS+vl5cWHnPMbKyZPWVmT1VwLKCZlewaPQMqxnsakD16BknSplkfIIQwSdIkidNoQFboGZAGXQOyR88aXyVnQF6RtP16ebvCYwCqi64B2aNnQPboGSRVtgCZI6mvmfUxs3aSRkiaUZ2xAKyHrgHZo2dA9ugZJFVwCVYIYa2ZnS7pfkltJE0OITxXtckASKJrQAr0DMgePcM/tPpjeFt1MK7jq3dPhxAG1noIFEfP6h49qxN0rb7xMbz1gZ7VvQ2+p3EndAAAAADJsAABAAAAkAwLEAAAAADJsAABAAAAkAwLEAAAAADJsAABAAAAkAwLEAAAAADJsAABAAAAkEyr74QOAM3ogQcecNnM38vsS1/6UspxgBbr16+fy0OHDnV57NixLs+ZM8flP//5z0Wf/2c/+5nLa9asKXNCAM2CMyAAAAAAkmEBAgAAACAZFiAAAAAAkmEPSBnatm3r8n777efyj3/8Y5e/+MUvZj4TgGz953/+p8tx72+55ZaU4wAtdsopp7h8+eWXu9yxY8eiv7/DDju4PGLEiKI/H+8ZmT17dqkRATQpzoAAAAAASIYFCAAAAIBkWIAAAAAASIY9IGXo3Lmzy/H1ra+99prLn/rUp4p+H0D+TJw40eV///d/d/mjjz5yOb4vCJAX//Vf/+Xyj370I5dL7QEp17Rp01w+9thjXZ45c2ZVjwegfnEGBAAAAEAyLEAAAAAAJMMCBAAAAEAy7AGponjPB3tAgPqzzz77uBzf/+exxx5z+c4778x8JqA13njjDZcvuOACl6+44gqXN998c5dfeukll3v27Fn0eFtttZXLhxxyiMvsAQHS69Wrl8sdOnRweeTIkS6feuqpRZ/vd7/7ncsnnnhiq+biDAgAAACAZFiAAAAAAEiGBQgAAACAZNgDUkVmVusRgLq3//77u/y9733P5fh61fg693LFz7fbbru5vHjxYpfPOeecio4H1Mr111/vcnyPmz322MPl1atXV3S8a6+9tqLfB1Da4MGDXT7yyCNdjt/j4nvahRDKOl68T7K1OAMCAAAAIBkWIAAAAACSYQECAAAAIBn2gFRRfB3dZpttVqNJgPo1adIkl/v27etyv379XI7vy1Gu8847z+VtttnG5ZNPPtnlv/zlLxUdD8iLiy++2OV4v1X//v0rev527dpV9PsApBtvvNHlz33ucy7vtddeZT3f22+/7fJtt93m8pw5c1y+/fbbXf7ggw/KOt7GcAYEAAAAQDIsQAAAAAAkU3IBYmaTzWyFmc1d77EuZjbLzBYW/tw62zGBxkfXgOzRMyB79AyltGQPyBRJ10q6Zb3HJkh6IIQw0cwmFPL46o9X3wYOHOjyE088UaNJUCemiK7pvffec7nae6vi69p79erl8rp166p6POTOFNEzSdJvfvMbl+P9VDNnznQ5vva8lHiPyde+9rWyfh91bYroWYvE+w4vvfRSl7/xjW+4HN/76umnn3Z54sSJLs+dO9fl999/3+WXXnqp5cNWUckzICGERyTFd/oaJmlq4eupkoZXdyyg+dA1IHv0DMgePUMprf0UrG4hhGWFr1+T1G1jP2hmYyWNbeVxgGbXoq7RM6AivKcB2aNn+KeKP4Y3hBDMbKP3cQ8hTJI0SZKK/RyA4op1jZ4B1cF7GpA9eobWLkCWm1n3EMIyM+suaUU1h8qrtWvXuvzWW2+53LlzZ5d32GGHzGdCw2v4rl100UUux9eZz58/3+Vy78OxxRZbuDx+vL/kePPNN3c53qsVXyePhtTwPduQ4447zuU99tjD5d12262i56/0Hj1oOE3Zs1K+//3vu3zSSSe5fM0117gc36/nnXfeyWawjLX2Y3hnSBpd+Hq0pOnVGQdAhK4B2aNnQPboGf6pJR/De7ukxyXtbGZLzewkSRMlDTGzhZIGFzKACtA1IHv0DMgePUMpJS/BCiGM3Mi3DqryLEBTo2tA9ugZkD16hlIq3oTeTFatWuXyo48+6vLQoUMTTgPUp+23397lk08+2eV4r9Xpp5/u8sqVK8s63pVXXuny0Ucf7fKrr77q8he/+MWynh/Iq1122cXlu+++2+Udd9zR5U03re5fCWbMmFHV5wPqQbyvMN53OGrUKJfPPPNMl2fPnu3y/fff7/IHH3xQ4YT50No9IAAAAABQNhYgAAAAAJJhAQIAAAAgGfaAAMhUfC+B+Dr0rl27uhx/5vnDDz9c1vHOOeccl8eMGVP05y+55JKynh+oF7vuuqvLffr0cbnaez5iZ511lstnnHFGpscD8uD88893Od4Dcuedd7o8c+ZMlxtlj0cpnAEBAAAAkAwLEAAAAADJsAABAAAAkAx7QDK0zTbb1HoEIHPxdeTHH3+8yzfddJPLm2zi/7vHunXrXN53331d/u53v+tyfF+PLl26uBzf58PMXL7llltcvuGGGwQ0oni/1Xe+8x2XL7vsMpc322yzqh6/e/fuVX0+oB7E71khBJdvv/12l5tlz0eMMyAAAAAAkmEBAgAAACAZFiAAAAAAkmEPSIaOOOKIWo8AZG7EiBEu33jjjS7H17/Gez4WLVrk8sCBA4vmYcOGudyjRw+X4+vOV65c6fI3vvENAc3o6quvdnnhwoUub7XVVkV/P97vde2117rcqVOn1g8HNIj/+Z//cTl+D4t78/7777s8a9asbAbLGc6AAAAAAEiGBQgAAACAZFiAAAAAAEiGPSAVmD17tstDhw6t0SRAOscee6zLN998s8sfffSRy6tWrXL561//ustvvvmmy1dccYXLBxxwgMvx9bTxfT7iPSddu3Z1+eWXX3b5wAMPdHnx4sUCmsF9991X1s/HXdtxxx1d/sEPfuBy//79Xe7Vq5fLS5YsKev4QC184QtfcPnPf/6zy2vWrHH50EMPdfnb3/62y9///vdd/s1vflP0eAsWLGj5sHWEMyAAAAAAkmEBAgAAACAZFiAAAAAAkmEPSAVeeumlot9v27aty1z/ikZwyimnuBz34OKLL3Y53iNSyhlnnOHyDTfc4PK+++5b1vPF163He7fY8wG0TLt27VyO93zE4v1gH3/8cdVnAioV3zvq3nvvdblnz54un3XWWS7feuutLr/xxhsux/f9iPeAdOzY0eUuXbqUmLgxcAYEAAAAQDIsQAAAAAAkwwIEAAAAQDLsAanA2rVri34/vva8ffv2WY4DJDF9+nSXp02b5nJ8n41yxfft2G233Yr+/MiRI12eO3du0Z9funRp6wYDmly8v6uUm266yWW6hzx65plnXO7UqZPL48ePdzne81HKuHHjin7/D3/4g8ul3sMaBWdAAAAAACTDAgQAAABAMixAAAAAACRjIYR0BzNLd7AamDdvnsu77LKLy9dff73Lp512WuYzVdnTIYSBtR4CxdVbzzp37uxyfJ153JP4vh077bRTNoPVDj2rE1l3bZtttnE5vqfO7bffXjRXKr4/woIFC1yOr5WP7bDDDi6/+OKL1RmsSkIIVvqnUGtZ9+y73/2uy+eff77LHTp0KOv5Fi5c6HLfvn1dju8Bd9RRR7kc70lpABt8T+MMCAAAAIBkWIAAAAAASKbkAsTMtjez2WY2z8yeM7Nxhce7mNksM1tY+HPr7McFGhM9A9Kga0D26BlKacl9QNZKOjuE8IyZbSnpaTObJWmMpAdCCBPNbIKkCZLGF3mehjdz5kyXe/To4fJ//Md/pBwH9aVpexbv8Tj11FNdXrFihctf+tKXMp8JDa1uunb11Ve7fPjhh7sc73969dVXXX7llVdcXrRokct77rln0ef7zne+43KpPR9XXHFF0XnQVOqmZ5deeqnLH330kcsDBgxwefDgwUWfb+ut/Zrqd7/7ncvnnHOOy3Evm0XJMyAhhGUhhGcKX78tab6kHpKGSZpa+LGpkoZnNCPQ8OgZkAZdA7JHz1BKWXdCN7PekgZIelJStxDCssK3XpPUbSO/M1bS2ApmBJoKPQPSoGtA9ugZNqTFm9DNrKOkuySdGUJYvf73wief5bvBj0kLIUwKIQzkYyWB0ugZkAZdA7JHz7AxLToDYmZt9ckL6LYQwrTCw8vNrHsIYZmZdZe0YuPP0Jzie6ysWbOmRpOgHjRLz3r16uXyN7/5TZfj3kyaNMnlpUuXZjMYmka9dO2aa65xuU+fPi7vu+++Lj/00EMu/+1vf3M5vlfVv/7rv7q85ZZbFp0n7mZ8X5ALLrjA5Q8++KDo86Gx1UvPYpdffnmtR2gKLfkULJN0k6T5IYQr1/vWDEmjC1+PljS9+uMBzYGeAWnQNSB79AyltOQMyBcljZL0VzN7tvDYeZImSrrTzE6StETSMZlMCDQHegakQdeA7NEzFFVyARJCeEySbeTbB1V3HKA50TMgDboGZI+eoZSyPgUL5Yk/M33YsGEu33333SnHAXJh1qxZLsd7Qm699VaX4+vKgWbxxBNPuPz444+7/Ktf/crl6667zuXevXsXzeV68803Xe7Xr19FzwegebX4U7AAAAAAoFIsQAAAAAAkwwIEAAAAQDLsAamiY47xH+bw4Ycfujx//vyU4wC5dPPNN7t80UUXuTx9Op/KCGzI2Wef7XL79u1d7tixY9HfHzBggMsjR44s+vNvvfWWy0OGDCk1IgC0CGdAAAAAACTDAgQAAABAMixAAAAAACRjIYR0BzNLd7AauOOOO1zeddddXT7iiCNcXrJkSeYzVdnTIYSBtR4CxTV6z5oAPasTdK2+hRA2dqM85Ag9q3sbfE/jDAgAAACAZFiAAAAAAEiGBQgAAACAZLgPSBWNGDGi1iMAAAAAucYZEAAAAADJsAABAAAAkAwLEAAAAADJsAABAAAAkAwLEAAAAADJsAABAAAAkAwLEAAAAADJsAABAAAAkAwLEAAAAADJsAABAAAAkAwLEAAAAADJbJr4eK9LWiKpa+HrvGK+DetVg2OifPSsOugZSqmHruV5NomeobR66JnEfBuzwa5ZCCH1IDKzp0IIA5MfuIWYD40g768T5kOjyPNrJc+zSfmfD/mR99cK85WHS7AAAAAAJMMCBAAAAEAytVqATKrRcVuK+dAI8v46YT40ijy/VvI8m5T/+ZAfeX+tMF8ZarIHBAAAAEBz4hIsAAAAAMmwAAEAAACQTNIFiJkdYmbPm9kiM5uQ8tgbY2aTzWyFmc1d77EuZjbLzBYW/ty6RrNtb2azzWyemT1nZuPyNB/yK29dy3PPCrPQNZSNnpU9Hz1D2fLWMynfXauXniVbgJhZG0k/l3SopH6SRppZv1THL2KKpEOixyZIeiCE0FfSA4VcC2slnR1C6CdpH0nfKvwzy8t8yKGcdm2K8tszia6hTPSsVegZypLTnkn57lpd9CzlGZC9JS0KIbwYQlgj6Q5JwxIef4NCCI9IeiN6eJikqYWvp0oannKmfwghLAshPFP4+m1J8yX1yMt8yK3cdS3PPZPoGlqFnpWJnqEVctczKd9dq5eepVyA9JD08np5aeGxPOoWQlhW+Po1Sd1qOYwkmVlvSQMkPakczodcqZeu5fJ1TNfQQvSsAvQMLVQvPZNy+DrOc8/YhF5C+ORzimv6WcVm1lHSXZLODCGsXv97eZgPqFReXsd0DY0sL69heoZGl4fXcd57lnIB8oqk7dfL2xUey6PlZtZdkgp/rqjVIGbWVp+8gG4LIUzL23zIpXrpWq5ex3QNZaJnrUDPUKZ66ZmUo9dxPfQs5QJkjqS+ZtbHzNpJGiFpRsLjl2OGpNGFr0dLml6LIczMJN0kaX4I4cr1vpWL+ZBb9dK13LyO6RpagZ6ViZ6hFeqlZ1JOXsf10rOkd0I3s8Mk/UxSG0mTQwiXJDv4RpjZ7ZIOlNRV0nJJF0i6R9KdknpKWiLpmBBCvNkoxWyDJD0q6a+S1hUePk+fXMtX8/mQX3nrWp57VpiPrqFs9Kzs+egZypa3nkn57lq99CzpAgQAAABAc2MTOgAAAIBkWIAAAAAASIYFCAAAAIBkWIAAAAAASIYFCAAAAIBkWIAAAAAASIYFCAAAAIBk/h9TRDbgoUnDRgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x504 with 12 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(15, 7))\n",
    "for i in range(12):\n",
    "    plt.subplot(3, 4, i + 1)\n",
    "    plt.imshow(trainX[i], cmap=plt.get_cmap('gray'))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "79b3530d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, ..., 5, 6, 8], dtype=uint8)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32cf8e67",
   "metadata": {},
   "source": [
    "## Process data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "de2a579d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28, 28)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainX[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "203077b6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   3,\n",
       "         18,  18,  18, 126, 136, 175,  26, 166, 255, 247, 127,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,  30,  36,  94, 154, 170,\n",
       "        253, 253, 253, 253, 253, 225, 172, 253, 242, 195,  64,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,  49, 238, 253, 253, 253, 253,\n",
       "        253, 253, 253, 253, 251,  93,  82,  82,  56,  39,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,  18, 219, 253, 253, 253, 253,\n",
       "        253, 198, 182, 247, 241,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,  80, 156, 107, 253, 253,\n",
       "        205,  11,   0,  43, 154,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,  14,   1, 154, 253,\n",
       "         90,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 139, 253,\n",
       "        190,   2,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  11, 190,\n",
       "        253,  70,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  35,\n",
       "        241, 225, 160, 108,   1,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         81, 240, 253, 253, 119,  25,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,  45, 186, 253, 253, 150,  27,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,  16,  93, 252, 253, 187,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0, 249, 253, 249,  64,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,  46, 130, 183, 253, 253, 207,   2,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  39,\n",
       "        148, 229, 253, 253, 253, 250, 182,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  24, 114, 221,\n",
       "        253, 253, 253, 253, 201,  78,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,  23,  66, 213, 253, 253,\n",
       "        253, 253, 198,  81,   2,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,  18, 171, 219, 253, 253, 253, 253,\n",
       "        195,  80,   9,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,  55, 172, 226, 253, 253, 253, 253, 244, 133,\n",
       "         11,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0, 136, 253, 253, 253, 212, 135, 132,  16,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0],\n",
       "       [  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0]], dtype=uint8)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainX[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7480537b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   3,  18,  18,  18,\n",
       "       126, 136, 175,  26, 166, 255, 247, 127,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,  30,  36,  94, 154, 170, 253,\n",
       "       253, 253, 253, 253, 225, 172, 253, 242, 195,  64,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,  49, 238, 253, 253, 253,\n",
       "       253, 253, 253, 253, 253, 251,  93,  82,  82,  56,  39,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  18, 219, 253,\n",
       "       253, 253, 253, 253, 198, 182, 247, 241,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "        80, 156, 107, 253, 253, 205,  11,   0,  43, 154,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,  14,   1, 154, 253,  90,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0, 139, 253, 190,   2,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,  11, 190, 253,  70,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  35,\n",
       "       241, 225, 160, 108,   1,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,  81, 240, 253, 253, 119,  25,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,  45, 186, 253, 253, 150,  27,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,  16,  93, 252, 253, 187,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0, 249,\n",
       "       253, 249,  64,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  46, 130,\n",
       "       183, 253, 253, 207,   2,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  39, 148,\n",
       "       229, 253, 253, 253, 250, 182,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  24, 114,\n",
       "       221, 253, 253, 253, 253, 201,  78,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  23,  66,\n",
       "       213, 253, 253, 253, 253, 198,  81,   2,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  18, 171,\n",
       "       219, 253, 253, 253, 253, 195,  80,   9,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,  55, 172,\n",
       "       226, 253, 253, 253, 253, 244, 133,  11,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "       136, 253, 253, 253, 212, 135, 132,  16,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0], dtype=uint8)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp = trainX[0].reshape((28 * 28))\n",
    "temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "18b80378",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X_processed = []\n",
    "for elem in trainX:\n",
    "    train_X_processed.append(elem.reshape((28 * 28)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "559c5e8f",
   "metadata": {},
   "source": [
    "## Trying models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af0eec1f",
   "metadata": {},
   "source": [
    "### Random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c090b4d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train_X_processed\n",
    "y = trainy\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)\n",
    "\n",
    "clf = RandomForestClassifier(max_depth=5, n_estimators=10).fit(X_train, y_train)\n",
    "\n",
    "y_pred = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "485ab860",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8093434343434344"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17f0038d",
   "metadata": {},
   "source": [
    "### Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8953e460",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train_X_processed\n",
    "y = trainy\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)\n",
    "\n",
    "clf = XGBClassifier(max_depth=3, n_estimators=5, verbosity = 0).fit(X_train, y_train)\n",
    "\n",
    "y_pred = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e4ab7c9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8386363636363636"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f293421a",
   "metadata": {},
   "source": [
    "## Processing test to obtain results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f9a15085",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['arr_0']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_test = np.load('test_mnist_x.npz')\n",
    "data_test.files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2229f9f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "testX = data_test['arr_0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e1a7f1cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_X_processed = []\n",
    "for elem in testX:\n",
    "    test_X_processed.append(elem.reshape((28 * 28)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "4d21e0bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_classifier = XGBClassifier(max_depth=3, n_estimators=5, verbosity = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "1f5862a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train_X_processed\n",
    "y = trainy\n",
    "\n",
    "clf = best_classifier.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d2567568",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = clf.predict(test_X_processed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "ddb8f95e",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = pd.DataFrame(y_test, columns=['Category'])\n",
    "res.index.name = 'Id'\n",
    "res.to_csv('mnist_submission.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09ce1f4b",
   "metadata": {},
   "source": [
    "## Hints"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8592648b",
   "metadata": {},
   "source": [
    "$1$. Попробуйте другие алгоритмы, которые мы обсуждали на лекции: $kNN$, $Linear~regression$.\n",
    "\n",
    "$2$. Попробуйте подобрать лучшие параметры моделей, исследуйте документацию, чтобы узнать какие еще есть параметры.\n",
    "\n",
    "$3$. Попробуйте создать доп парметры: число замкнутых областей на картинке, доля черного и т.п., проявите фантазию!\n",
    "\n",
    "$4$. Возможно есть смысл попробовать сделать преобразования картинки, можно попробовать что-то из такого: усреднить цвета соседних пикселей, нормализовать цвета, снизить как-нибудь разрешение, что угодно еще."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}