trying_out_jax.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Trying Out Jax\n",
        "Note: all cells outside the initial setup section are written so they can be executed in isolation."
      ],
      "metadata": {
        "id": "u5qH_U3eWQLO"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Initial Setup"
      ],
      "metadata": {
        "id": "opyQqF3YEGFI"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "id": "pqZCfXoCwjPJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a72bc6cd-e6fb-4685-d31d-8268b62b50fb"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[StreamExecutorGpuDevice(id=0, process_index=0, slice_index=0)]"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ],
      "source": [
        "import jax\n",
        "import jax.numpy as jnp\n",
        "import numpy as np\n",
        "from jax import jit, value_and_grad, grad, vmap\n",
        "from jax._src.api import block_until_ready\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Only using these for data loading.\n",
        "import torch\n",
        "from torchvision.datasets import MNIST\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "jax.devices()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def custom_transform(x):\n",
        "    x = np.ravel(np.array(x, dtype=np.float32))\n",
        "    return x\n",
        "\n",
        "def custom_collate(batch):\n",
        "    transposed_data = list(zip(*batch))\n",
        "    labels = np.array(transposed_data[1])\n",
        "    labels = np.eye(10)[labels]\n",
        "    images = np.stack(transposed_data[0])\n",
        "    return images, labels\n",
        "\n",
        "def load_mnist(batch_size):\n",
        "    transform = transforms.Compose([\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize((0.5,), (0.5,)),\n",
        "        custom_transform,\n",
        "    ])\n",
        "\n",
        "    train_dataset = MNIST(root='data/', train=True, download=True, transform=transform)\n",
        "    test_dataset = MNIST(root='data/', train=False, download=True, transform=transform)\n",
        "\n",
        "    train_loader = torch.utils.data.DataLoader(\n",
        "        train_dataset, \n",
        "        batch_size, \n",
        "        shuffle=True, \n",
        "        collate_fn=custom_collate\n",
        "    )\n",
        "    test_loader = torch.utils.data.DataLoader(\n",
        "        test_dataset, \n",
        "        batch_size, \n",
        "        shuffle=True, \n",
        "        collate_fn=custom_collate\n",
        "    )\n",
        "\n",
        "    return train_loader, test_loader"
      ],
      "metadata": {
        "id": "ny7qy0nnczDK"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Updating Arrays Functionally"
      ],
      "metadata": {
        "id": "UNxTOorEy78M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Arrays are immutable in Jax, so x[0] = y won't work.\n",
        "# Keys are also passed to random.normal() explicitly instead of relying on explicit state.\n",
        "key = jax.random.PRNGKey(100)\n",
        "x = jax.random.normal(key=key, shape=(5,))\n",
        "print('Original x:', x)\n",
        "x = x.at[0].set(20.0)\n",
        "print('Updated x:', x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MMGKN4csw98N",
        "outputId": "f5f3c066-5743-4e60-e175-166adfae3f13"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original x: [-0.69846076 -0.7054794   1.3151264  -2.0337505  -0.08158301]\n",
            "Updated x: [20.         -0.7054794   1.3151264  -2.0337505  -0.08158301]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "kOLkx_PKWhf_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Speed Test"
      ],
      "metadata": {
        "id": "0DpD1bqd0Frs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "key = jax.random.PRNGKey(100)\n",
        "x = jax.random.normal(key, shape=(1000, 1000))\n",
        "key = jax.random.PRNGKey(200)\n",
        "y = jax.random.normal(key, shape=(1000, 1000))\n",
        "\n",
        "def numpy_matmul(x, y):\n",
        "    return np.dot(x, y)\n",
        "\n",
        "def jnp_matmul(x, y):\n",
        "    return jnp.dot(x, y)\n",
        "\n",
        "# Just-in-time (JIT) compilation can be used to speed up execution.\n",
        "jnp_matmul_compiled = jit(jnp_matmul)\n",
        "\n",
        "print(\"np matmul:\")\n",
        "%timeit numpy_matmul(x, y)\n",
        "print(\"jnp matmul:\")\n",
        "%timeit jnp_matmul(x, y).block_until_ready()\n",
        "print(\"jnp matmul compiled:\")\n",
        "%timeit jnp_matmul_compiled(x, y).block_until_ready()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VMeiKp6hxoSH",
        "outputId": "9948d75e-6a94-4dfd-a4e6-0c7fa0835fd2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "np matmul:\n",
            "28.3 ms ± 1.21 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n",
            "jnp matmul:\n",
            "1 ms ± 18.3 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n",
            "jnp matmul compiled:\n",
            "576 µs ± 3.06 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## FEBAM/Hebbian Learning\n",
        "https://ieeexplore.ieee.org/document/4371210d\n",
        "\n",
        "https://github.com/hayden-donnelly/febam"
      ],
      "metadata": {
        "id": "PFdjsDCXUOXG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def create_febam(key, x_dim, y_dim):\n",
        "    W_key, V_key = jax.random.split(key, 2)\n",
        "    W = jax.random.normal(W_key, shape=(y_dim, x_dim))\n",
        "    V = jax.random.normal(V_key, shape=(x_dim, y_dim))\n",
        "    return (W, V)\n",
        "\n",
        "def sigmoid(x):\n",
        "    return 1/(1   jnp.exp(-x))\n",
        "\n",
        "def execute_febam(weights, x0):\n",
        "    W = weights[0]\n",
        "    V = weights[1]\n",
        "    y0 = sigmoid(jnp.dot(W, x0))\n",
        "    x1 = sigmoid(jnp.dot(V, y0))\n",
        "    y1 = sigmoid(jnp.dot(W, x1))\n",
        "    return x1, y1, y0\n",
        "\n",
        "def update_weights(weights, x0, xt, y0, yt, lr):\n",
        "    W = weights[0]\n",
        "    V = weights[1]\n",
        "    W_diff = lr * jnp.outer(y0 - yt, x0   xt,)\n",
        "    V_diff = lr * jnp.outer(x0 - xt, y0   yt,)\n",
        "    return (W   W_diff, V   V_diff)\n",
        "\n",
        "# Set hyperparameters\n",
        "epochs = 50\n",
        "lr = 0.005\n",
        "batch_size = 1\n",
        "x_dim = 784\n",
        "y_dim = 200\n",
        "\n",
        "# Load data.\n",
        "key = jax.random.PRNGKey(1)\n",
        "train_loader, _ = load_mnist(batch_size)\n",
        "image = next(iter(train_loader))[0].squeeze()\n",
        "\n",
        "# Create model.\n",
        "febam_weights = create_febam(key, x_dim, y_dim)\n",
        "\n",
        "# Compile inference and learning functions.\n",
        "execute_febam_c = jit(execute_febam)\n",
        "update_weights_c = jit(update_weights)\n",
        "\n",
        "# Train the model.\n",
        "x_throughout_training = [image.reshape(28, 28)]\n",
        "x0 = image\n",
        "xt, yt, y0 = execute_febam_c(febam_weights, x0)\n",
        "febam_weights = update_weights_c(febam_weights, x0, xt, yt, y0, lr)\n",
        "x_throughout_training.append(xt.reshape(28, 28))\n",
        "\n",
        "for epoch in range(epochs-1):\n",
        "    xt, yt, _ = execute_febam_c(febam_weights, xt)\n",
        "    febam_weights = update_weights_c(febam_weights, x0, xt, yt, y0, lr)\n",
        "\n",
        "    if epoch % 10 == 0:\n",
        "        x_throughout_training.append(xt.reshape(28, 28))\n",
        "\n",
        "fig, axs = plt.subplots(1, len(x_throughout_training),  figsize=(12, 12))\n",
        "for i, img in enumerate(x_throughout_training):\n",
        "    axs[i].imshow(img, cmap='gray')\n",
        "    axs[i].axis('off')\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "ROtc69Bs18h7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 191
        },
        "outputId": "6055ef06-18ad-4251-883c-cf3cfd7c5c06"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1200x1200 with 7 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAACuCAYAAADnNb6bAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAtu0lEQVR4nO3dd5RV1fn/8W1oAwx1kN6RKggCIl1RiooNMVhRbEFs0ahERWMjNhBUFhoLWMCCywgWimCEBUEFUUFAQRFFQerg0MtA P2Rle/KL/vzyNm3nLkzvF9/ftYz5 57zr7nXo s5znq0KFDhxwAAAAAAAAQo98V9AIAAAAAAABw5OGhFAAAAAAAAGLHQykAAAAAAADEjodSAAAAAAAAiB0PpQAAAAAAABA7HkoBAAAAAAAgdjyUAgAAAAAAQOx4KAUAAAAAAIDY8VAKAAAAAAAAsSsetfCoo45K5zqA/3Po0KGUHi9k75YoUcLL5s fL2tbtmzpZdbas7OzI9eqPOQ9pOL8LViwQOatWrXysu7du8vaRYsWeZn1PubNm dl3bp1k7Xq/T3yyCOy9o477vCyUqVKydr9 /d72b/ 9S9Zm477Ycgx1bp 9zv9/xhUbp3bOXPmeFnx4vprYuHChV52wgknyFr13g4cOCBro/79bylWrJiX3XTTTbK2Q4cOXjZ06FBZm5eX52U7duyQtfn5 V6m7i h1PVcv369rK1atWpCr9GjRw Zq/1hUe9VnRPLaaedJvMZM2ZEPkaINm3aeNmXX34pa7/77jsva9KkSaqX9Jusz0Sqvz9T5fTTT5f5tGnTEjoev0n/zfqMb9q0KeaVpIe631nfy mS6GeKPYq4JHPfZ58iLofbp/xLKQAAAAAAAMSOh1IAAAAAAACIHQ lAAAAAAAAEDseSgEAAAAAACB2kRudA4XV4sWLvez777 Xtf379/eyu   W9bOnj3by1SzbOd0Y06r4bZqOvjzzz/L2tq1a3uZavLsnHMHDx6UuXLiiSdGrlUNzS1WU jy5ctHPobyxRdfyDzZBvGlS5eWtXv27Il83P 1detWmatm7127dpW16n1ZDQRVbUjD6quuukrm7dq1i7yGqOtyzrnHHnss8nGt5u6qUfeoUaNkbYUKFbxs /btslY1RVcN353TDeL37t0ra7OysrwsZO8l2tDcctFFF8lcnauzzjpL1l5zzTVJraFv374yV43OrWb8qpn rl27ZG21atW8LKSZuNV82brPq/vxkiVLZK1qZN 7d29Zm6n 8Y9/FNhr16tXT Zr1qxJ6rjWQAD12yAV2rZt62XWd19OTo6XWfe1kAEEcYu7qXlRE/dAhGQHXAA4cvEvpQAAAAAAABA7HkoBAAAAAAAgdjyUAgAAAAAAQOx4KAUAAAAAAIDYHXUoYre7kIbBQDISbcBo7dEpU6Z42bnnnitrL7vsMi976aWXZK1qzr1z505ZO336dC877bTTZG3ZsmW9zGrOq1iNQVVT6GXLlsnaVq1aybxLly5eNn/ fFmbm5vrZar5qiWkYfvNN98sax955BEvK1mypKxV yfuJqFRX181Plb7xjnnsrOzvcxqEK6oZvrO6etet25dWaveh2qa7ZxzP/zwg5dVqlQp8nGd09fHaraqGrOGHLdXr16ydurUqV5mNb2O lrOOTd69Ggvsz6vPXv2jPx6/816/2pQQbly5RJ6jYLSvXt3mc dO9fLdu/eLWtVw/orr7xS1qrvH ecW7VqlZcdc8wxslbtsVmzZslatWbrGoUMvwiRl5fnZepz5pxzZcqUSeg1rHtYXPfmdLK o6xBKlG1adNG5moYTFGn7gPqHuBc6n TwlbQv7kySUjT GTOD/sUcTncPuVfSgEAAAAAACB2PJQCAAAAAABA7HgoBQAAAAAAgNjxUAoAAAAAAACx46EUAAAAAAAAYsf0PWScRKdI9OjRQ YjRozwsvbt20c rrX31eQia2JcCPX N23aJGurVasW6e dc 6EE07wss8  0zW3njjjTIfM2aMzBU1Ra1evXqyVp03630key yjqvWkI7pVNYeOXDggJeFTMkLETK5Ku6pN6effrqXTZs2TdY 9NBDMr/rrru8LGSqz9atW2Vt5cqVvUyt1znnZsyY4WXW50d93kL2f6qvkbU/1Gtb0z6PNNaEVLVnnHPu2muv9bKxY8fK2l9  cXLmjRpImvVd4U1zS1O1nlQU1qjsD7PcXw cGRh h4yHdP3UBgwfQ8AAAAAAAAZh4dSAAAAAAAAiB0PpQAAAAAAABA7HkoBAAAAAAAgdsULegFAqsyZM0fmjRo18rI6derI2rVr13pZxYoVZW1I01/VOHjBggWyVh3j6KOPlrVbtmyJtC7ndIO5kFqrvmfPnrL2ww8/lLkybNgwL vSpUvktVnrrVu3buTazZs3e9n7778va88880yZRxGyRyxVqlTxso0bN8rakOb7IU2r9 3b52VZWVmR/z6kMafVcF41NHcurHFnuhrnh6hfv35Sr1erVi1Zu27duoTWY70nlauBBs4516BBg4Re z s61K fHkvy87OlrWJvv/Dufzyy72sbNmysrZly5Yyf/LJJyO/XtWqVSPXqqbm1rm84YYbvEy9N f0enfu3ClrJ0 e7GXbtm2TtamWCU3Na9So4WXr168vgJX8/6zvGOs7Ij8/P53LiaR06dJepoaCOJcZ6wUAJIZ/KQUAAAAAAIDY8VAKAAAAAAAAseOhFAAAAAAAAGLHQykAAAAAAADEjodSAAAAAAAAiN1RhyKOKkl2QhEQVaqn56jjWVNoQibURf370GOoSTjWtBk1lU9NkXNOT2Xav3 /rM3JyZF5bm6ul23dulXWNm3a1MvUtMBQlStX9rK8vDxZ27ZtWy/77LPPZK06xyVKlJC1BT3hqXhxf3CqNaFOsfa/mr63adMmWav2nnUd1HQ069wOHjzYy6z1du3aVeYXX3yxl23fvl3WVqhQwcvU3nXOuZUrV8pcUfupZs2asnbp0qVeZk1cU1MWrc r2idRHH/88TL/4osvvCxkamTc5s2b52XdunWTtR06dPCyhQsXJr2Ghg0bylzdm956662kXy9EyJQ49R326KOPytqhQ4dGXkOi91LrO1V97hL9HMShdu3aXqamAKPgpHqPAqmWzG9S9inicrh9mrm/JgEAAAAAAFBk8VAKAAAAAAAAseOhFAAAAAAAAGLHQykAAAAAAADELnO7P6aJauh24YUXytqKFSt62bPPPitrTznllEiZc87deeedMn/  ee9rHHjxrL2oosu8jLVANe5gm/KXNDUNf/2228j11pKlSrlZVaTcXUNrOblqgG01ch4 PDhXmY1L7dyZdasWTJXTY2ttan9qJq6Omc311VUw2rVoNs552699VYvC2niHie190IaklvUXm/SpImsDWn0n5 f72WVKlWKvC6r fBVV13lZe3atZO1aiiAc85dcsklXnbqqafKWvWereOGCGmubDU1V6pVq Zlqb7HWw3rM6GpuVqD9XlQTewtyTY1D7m/OqcHT7Rs2VLWli1b1ssWLFgQ9HpKyH1XDTZQzdqdcy4rK8vL9u7dG31hScjkpuaKamqeivu 2l/lypWTtdYAEnXvb9Gihaz9 uuvI68tXcqXL 9l1oALJMf6XRDyXRTym8e6jure2KxZM1m7YsWKyGsDfou6vzrn3OzZs2XepUsXL7vttttk7ciRIxNfWCFX8L8wAQAAAAAAcMThoRQAAAAAAABix0MpAAAAAAAAxI6HUgAAAAAAAIgdD6UAAAAAAAAQu6MORRyVEDKVLJNVrlzZy 644w5Zq6byzZkzR9aqaU8HDx6UtXv27JF5mTJlvCxk2tHVV18t8xdffDHyMTJBopOkmjZtKvNVq1Z5mXVtlD/84Q8yf edd7zMmrxw7LHHepn1PtVUM4ua1GdRn2FrDc2bN5d5stNLrPem3of6TDnn3JNPPull6rPjnJ4CZX2m1BpmzJgha63JmlGkYmrN4sWLvez444 PfFyrVuXjx4 XtWramJpI6ZxzW7du9TLr86omWKZiulz9 vVl/uOPP3qZdY3UZ3716tWy9uabb458XPX rH0aci4SPW/WOtXEmQ8  EDW9ujRI6HXToQ1lSzk /PLL7/0sjZt2sjapUuXepk1OS/kt1O9evVkvmbNmsjHTdfkO7WXrPuu o2T6gmR1vsP Z4rbHbv3u1lAwYMkLV33323lzVq1EjWVqlSJfIarEmon3/ uZel4hqFTNtMlrXeRF vqPx3Uwj1Hfn73/9e1qr7mjXFtG7dupHX8Omnn8q8Y8eOXlZUrlEy97iicg5CDBkyxMteeeUVWaum1O7YsUPW1qxZM/Ia1G8O55xr1aqVl4X8t14mO9w 5V9KAQAAAAAAIHY8lAIAAAAAAEDseCgFAAAAAACA2PFQCgAAAAAAALE74hqdK5dffrnMVaPIhg0bytply5Z52X333SdrJ0 eLPN58 Z5WefOnWWtsnz5cpl37drVy7Zv3x75uHFLtGGf1QjuwIEDSb12SPPLkM9JSK3VQHrnzp1e1qlTJ1k7cuRILzvppJNkrRoI4Jxzubm5Xla9enVZu27dOi8rXry4rFXnwmp4qZotW5o1a ZlVmNq6/WUZJpKWucgpPm ev3p06fLWtWAVt3bnNMNya29oJocq0a8zqXv yPkOpQrV07mqmFlyHqt61mhQgUv27Jli6xV9yhrn6t9koqm6P8tFddr/vz5Xta9e3dZq 6lVnPh8uXLe5m179S5qlGjhqxVe3/fvn2yNoQ1KGPMmDFeFnJvs4ZGhBwjzibSloLco pchXwPWPfR4cOHJ7ym3/Lqq696mTUQRN1/tm3bJmuHDh0q88cee8zLrN9TxxxzjJf98MMPsraw/fdEQe7RwmbJkiVeZg2MUN8RIf/N45xzHTp08LJZs2bJ2nvvvdfLRo8eLWsL27Wj0XkYNeBGZc459/PPP3vZwIEDg17vgQce8LIbb7xR1qpBRtZ/16nBJpmMRucAAAAAAADIODyUAgAAAAAAQOx4KAUAAAAAAIDY8VAKAAAAAAAAseOhFAAAAAAAAGJ3xE3fa9SokZdZU6tU7dKlS2XtmWee6WVr164NWpua7Ddt2jRZ27hx48jHrVWrlpdt2LAh sJilugUia   krm6rxmZ2fL2rJly3rZnj17ZG1OTo6XWdO1FGvKkZqIZE3XUtOlrrjiClm7efNmL5s6depvLTGSZKZ /Ie6v6TiuJkwXep/WffSCRMmeNmll14qaydOnOhl1jSQKVOmeNm5554ra9U5t/aeyq3JVa 99pqXWe tefPmXvbee /J2pD7oLWf1JqtaZfJsiYszpgxw8v69u0ra1PxuTickImjFmvfRH29dL1P67jqvVnvIROmSVqSnRwZMq22X79 Mu/Vq5eXqXuAc3rqcBRx/yZVvw127dqVltdSE0Gds3 3KGrvpmJ/5eXlyVxN 7Nk6n9PWBNaE51WnanvMxXUJGbn9D4oVqyYrE3X SnKezQV381Rj1kUqMl5zunPurU/0nV Lr/8cpm/9NJLkY9R2K4d0/cAAAAAAACQcXgoBQAAAAAAgNjxUAoAAAAAAACx46EUAAAAAAAAYldkG52rJuXO6abmVu1PP/3kZd27d5e1VjO1ZA0ePFjmTz/9tJdZDUPPOOMML9u9e3dyC0ujRJtxhuzRkNf49ttvZd6sWTMvsxplbtu2zcuGDx8ua    24vsxokjxs3zsusBtKqUavVyNdqdqvyEiVKyFp1PVq3bi1rFy9eHOnvnQtrxq2aP4bsk4JuKmnt02XLlnmZdW7V3unTp4 sfeONN7yscuXKsnbfvn1eZjUIV8MCsrKyZG26WOddfd4efPDBpI rWNcz5Bhqr1v3h0Rlwve9tYabbrrJyy688EJZ27FjRy zmu62atXKy1avXi1r1T5/ eWXZe2kSZNk/sorr8hcUY2Wre8aNXQlFQMtkmU1kbWaEh9OJuzRVFD3hI8  kjWqv1sNZxu06aNl23dujVscYL1 VG/DazBF q3SEiD/bjF8Zs0k6n3b 07dR1r1qwpa MeOqPex86dO2WtdX/NVMkMMSgq 1QNiLAGjn3zzTdedt5556V8TYlQ19L6zT5z5sx0LyelaHQOAAAAAACAjMNDKQAAAAAAAMSOh1IAAAAAAACIHQ lAAAAAAAAEDseSgEAAAAAACB2xQt6AalQvLj/Nh566CFZa03aU8aPH 9l6Zqy55xzvXv39rKHH3448t/XqFFD5iVLlvSyTJ6 lyhrKsyAAQMiH2PatGlepqYXOqenCFjTbdTavvvuu8jrUnvcYk1sVKypXdY0jpApaur8WO/j5JNP9jJrop5aW9WqVWWter2bb75Z1n7yySdetmDBAlmbDiHTU9q3b 9lIdfSOrdq0p61rvz8fC9r166drC1dunSkdVmvp6aPOedc fLlZa78 uuvMlfvw6I x9b0SfU atWqFbnWOj/WFJY4/OUvf/GyBx54IPLfN23aVOYrV670MmvfPfnkk172xBNPRF6DdQ2WLFkS Rhqyk 3bt1kbd  fSMf13rPIVOqkp20t3z5cpm3aNHCy6w9mpOT42WpmPxWmFnX9p577vGyP/3pT7JWTa2L29KlS2Wu3p91b0zXpD21H5OZSnakUb/vLJUqVZK59Ts8TiHTaEN hyMzWP99n52d7WVqSrpzzjVv3jyla0pEyL3plltukXlhm753OPxLKQAAAAAAAMSOh1IAAAAAAACIHQ lAAAAAAAAEDseSgEAAAAAACB2RaLR dFHH 1l559/fuS/t5qXq0bnqVCuXDmZjx492ssqVKgQ biquahzYU2yCzOrueHrr7/uZXfddZesVU1prWZ01apV8zJrL6lGtSFN9y1vvfWWl33//fey9s9//rOXWU2/rffcsWNHL9u3b5 sVU1H582bJ2tVk2Cr4W79 vW9bNKkSbL2iiuu8DL1Hpxz7r333vMyNSQgXdT5CmkGvnPnzsivFdI42VpDbm6ul1mN4VWDXmvfNGnSxMushubWPv3ll1 8rGLFirJWNd197bXXZK06b9Z9R12PGTNmyFp1juvWrStr1TCGkH2SDNXUXDWxd043zf32229Tup7/sN7/888/72Vr1qyRtccdd5yXLVu2LPIarD16zDHHyDykwe6GDRu8rFOnTpH/3qL2hzUEIWQvqXvDka5OnToynzNnjpelqxG4xfr87Nq1y8usz7u6N4b8frWaou/du9fLrGbaNDVPzk033STz5557zssyZVhS//79vcy6h6k1q//uQma78sorZa4 / PGjUv3ciIZM2ZM5Fr129G67xY1/EspAAAAAAAAxI6HUgAAAAAAAIgdD6UAAAAAAAAQOx5KAQAAAAAAIHY8lAIAAAAAAEDsisRYtgEDBiT191Z3/nXr1iV1XGsazwsvvCDzZs2aJfV6b7/9tsy3bNmS1HELu5CJLCG1aiJSKqZgnX322V42ZcoUWZuVleVl1uSekDVY70OxpuYoXbt2jVxrTYhTk9xU5pyevmdNiFNTPNPBmqIRMs1GTZexzkGIkD2iJoSsXr1a1qoJddbUOvXeQieI1qpVy8us93bsscd6WSqmxKkpq9Ya1Hp//PFHWTto0KDIx42DtW/VfaF27dqy1nqvSsh73bFjh5dZk5meeOIJL1N7wznnevXqFXkN1u8INdlz//79srZmzZpeZk2vDKHu89bvhSpVqiT1Wm3atEnq75NhfUfl5 en5fXUJMclS5bIWnUfjHticsg01uuuu07mTz/9tJeFTBG0roU1aQ pp6bsOaf3hzVFPF2s 77Krb2kfntNmDAhuYUhrdR3lJp8brnmmmtSuZzDUr85nHOuTJkyXmbdd9V38Mknn5zUugoL/qUUAAAAAAAAYsdDKQAAAAAAAMSOh1IAAAAAAACIHQ lAAAAAAAAELsi0ei8SZMmSf29atgbSjU1Vw1pnXOuf// Sb ekmxj9sKuX79 MleN8qymiarWaj6rGuaGNOG1mpmGNAe94IILvGz9 vWytlSpUl5mvbeQ99GgQQOZq bFlStXlrWqIblqFuucc48//riXqSarzoU1bFfX02rM/NNPP0U 7v/as2ePzNW5ycvLk7Xq lhNE0OuZcieVnnIvTgnJ0fm6nPRs2dPWWtdX vYyjfffONlVjPskDWEXCN1787Ozpa1u3bt8jKrGbDVTD6VQj5jP/zwg8xDmtur92qdV9UM3FrvzTffLHOlYcOGXmY1 bcawat1WGtT70MN2vitYyi5ubledtNNN0X e4taQ8WKFZM bqLS1dDcsnTpUi zrsvYsWPTvZyEqc/l9ddfL2vV7 i9e/emfE3O2QMIli9fnpbXK4rU8ATre2/r1q3pXk7CTj/9dC bNGmSrFWftVQMJ0qWdd5DBhAUVWqQ2dChQ2Xto48 mu7lHJb1u 2dd97xsjPOOEPWqt kmbBP48C/lAIAAAAAAEDseCgFAAAAAACA2PFQCgAAAAAAALHjoRQAAAAAAABix0MpAAAAAAAAxK5ITN9LZhKWc87df//9Mj/  OO9bNy4cbJWTSRJ15Q9y4svvhjr62WayZMny1xNs7MmGVSoUMHLSpYsKWvVZKuQCQnWZCw1dWrt2rWyVk2hmThxYuQ1PPjgg7LWeh/qXGzbtk3Wli1bNtIanHNuwoQJXmZNHlGTSqzpYyVKlPCyVatWyVq1T6xrn4xevXrJ/KOPPvIy6xxs3LjRy5555hlZe8MNN0ReW8ieVqzroKZfWRNnSpcu7WUffPCBrE3F5BE1qc aNqTOhVqvVfvJJ5/IWjUFMysrS9YqqZ7S07p1a5kvWbLEy6z3r6ZMWhP1FGtimvqcWrXq89u2bVtZ /nnn0deW9euXb3s   /j/z3zum9 /rrr8tadY7Vd5Vzzn3xxRde1r59e1kbMqUyhHpvc bMSctrZaIVK1Z4mTWZVH12rYmNt99 e Q1qP0cOql6 /btXqam6joXNjU4ZCKywpS95IVMMVWs70h1T7GurZpS Ze//CXyGpxz7oUXXvAy6zepmto2bNiwoNdLB6bs2d58800vCzlfH374oczVfwdce 21slZN9bv44osjr8E553r37u1l1ueiT58 kWuLGv6lFAAAAAAAAGLHQykAAAAAAADEjodSAAAAAAAAiB0PpQAAAAAAABA7HkoBAAAAAAAgdkcditjSPWQCU9zUhJ0RI0bI2pBJVCHUxJtFixbJWjW5xznnWrRokdQaGjRoIPNkpxPGLdEpAyGT79K1n61JVGrSmDUlr1 /fl5mnZOTTjrJy bOnStrr7jiCi8bP368rLXOj5qskpubK2vVhAxr4tqWLVsivZZzzl133XVeZk2eC6EmE6kpSs4lt3 sv1VTjerXr5/0cUPOrWLtPTX50ZqKNHz4cC zpveMGjXKy6zJTiHT3NSEO f0lLty5crJ2ubNm3vZp59 GnkNIRM3Q6R6MkvI/ramJT3wwAOpWs5hnXXWWTJ/9913vcy6R /cudPLrIl8HTp08LLs7OzIx7XUrFlT5ur9Pffcc7JWfd7Lly8va9Vvp2rVqslaNfEzhPWZUhPeorD2qJq4ak1njNOpp54qczUdaubMmbK2Z8 eXmZ9pyohE20tGzZskHn16tW9LF2/s6y9tGPHjrS8Xqp/k2aqb7/9VuaNGzf2sk2bNsnaqlWrJrUG9VrOObdy5UqZq71rTbnt1KmTlxW2a2RJ5jdAYTsH6j7onHOzZs3ysry8PFlbsWLFpNZg/Y5YvXq1zNX9ccGCBbK2Y8eOXlbYrpHlcPuUfykFAAAAAACA2PFQCgAAAAAAALHjoRQAAAAAAABix0MpAAAAAAAAxK5INDpXSpUqJfNTTjnFy1TjZOecW79 vZeNGzdO1qqGyKqJqHN2I2rVaNkybdo0Lzv//PNl7b59 yIfNxMk2rDPavS bds2L7OaLIfs8127dnlZmTJlZK1qMFq3bl1Zu27dOi zzona51ZDZ0U1eXbOuT179shcreO8886TtVOmTPEyq4muapZuNVT961//6mX16tWTtUOHDvWykM D1aTYamqcjGLFinnZmjVrZG2dOnW8zGpaf WVV3qZtZ/U3rGa4KqGwu 9956sPfPMM2WuqM g1aA35PP6z3/ U bdunXzMuv8qHuMalDvnL5G1tAJ9T6sNbRv397LrKEaibLOa8g6k2XdH9X3ckgja2u9Ku/Tp4 sVQ1VrUbWVtNq9bmyGvqrBvurVq2StYrVjF81VA2h9rhzzv3888 Rj5Ho/rF X2VCU/MQ6jvcas4bMrTC i2izJ8/X adO3f2MmuPqu9263dWnKzvL s7RTlSGp2HuPTSS2U ePBgL3vttddkbciAmqVLl8q8ZcuWXqZ 8zvn3MCBA73M s1S2BxJjc5DjB07Vub33nuvl1n7JuQ7Zd68eTJXA86s/84aNGiQl7355puR15DJaHQOAAAAAACAjMNDKQAAAAAAAMSOh1IAAAAAAACIHQ lAAAAAAAAEDseSgEAAAAAACB2xQt6AeliTdiaPn16pCwVevXqJfOQKXuWjz/ 2MsK25S9VLOmEanpK0899ZSsTXaS1FlnnSXzZs2aednatWtl7fbt273skUcekbUhk/YUa/rDxRdfLHM1RWXy5MlJrcESMjVHTWBxTk/TsCaNtGjRwstefPFFWduhQweZRxEy6aR27doyVxObrrjiClmrcmsNaj8dPHhQ1qoJhNaUvZD3HPIZrFy5sszV1Kf69esn/Xrp HuLdc769esXuTbRtVlTkYYMGeJl5cqVk7U7duzwMmud1atX9zJrUmH58uW9bPfu3bK2cePGXvbhhx/KWuv7OqqvvvpK5ps3b5a5mlZWvLj Saa 29T9yjnnvvvuOy9TU4edc27v3r1eZk0y3bhxo5dZU/YqVKjgZdZko0TFPWUv2cmTd955p8wffvhhL2vYsKGsVZMnU Huu  W UcffeRlDz74oKzNhEl7SsiUPUQ3ceLEoDxZc bMkbn67Tdy5EhZW1Qm7SG666 /PtbXs6YGK9b9vKhM2ksE/1IKAAAAAAAAseOhFAAAAAAAAGLHQykAAAAAAADEjodSAAAAAAAAiN1RhyJ2agxpVIt/sxqnzpgxI ljN23a1MusRt FTaLNea0GnPfcc4 XWQ2 y5Qpk9Br/4e19mLFinnZs88 K2sHDx7sZQcOHJC16nNpfVZVY1irsa51jJAG2/fff7 Xvf3227J2wIABXlalShVZqz5XM2fOlLWqIfnChQtlrZKOJtYrV66UuWrKrPaNc/r99u7dW9aqPW3tJ9XofO7cubK2e/fuMlfeffddLzv77LNlrTrnVvPUgQMHRl6DdS3VXl 6dKmsbdWqVeTXUw2uVZNu5/TnymrQaw0AUBLdv1Zz 65du3rZp59 mtBr/De1z601KCVKlJB5upphq/OqmoY751zp0qUjH8Oi9uhLL70kawcNGhT5uJkg0T3Kb9Lfpu7x1ve91QxYfdeec845srYoXw/2aHqoQSVWw/xXXnlF5up3iDWsoShfj2R qxbl8xK3FStWyFz990xOTo6sLcrX43D7lH8pBQAAAAAAgNjxUAoAAAAAAACx46EUAAAAAAAAYsdDKQAAAAAAAMSOh1IAAAAAAACIHdP30ojpe4lJ9aQTNa3Kmmyl5OXlybxChQqR1xAi2Ylv1hp  eUXL6tevbqstabkzZ8/38tGjRoVeR3We3v55Ze9zJoipY5hTalTE7ys86POxdq1a2Wt9XpRWK v3pdV269fPy zrtmSJUu8TE0ldE5P36tRo4asVftp9 7dsrZ27dpe9uuvv0ZegzVdLWSi3meffSZrTzjhhEh/b71es2bNZK2awhKyT5966ilZe8MNN3jZRRddJGsnTZok88Ox3r acKqmm1qsqY87d 70sooVK0Y riXkXhHynaCmEC5fvlzWWtOkQtSsWdPL1q9fL2tD7iMhQo67evVqL2vYsGHk40aRrt kIZ/9VEjX9VLUPnLOuXXr1iV9bHWPtj7vaiKs9d0RYurUqV7Wt29fWXv11Vd72QsvvCBrM22Pxq1WrVpeloo9oxx33HEyV79jQmXC9cjKyvIya3JrCKbvxcuaghwyNdhSlK8H0/cAAAAAAACQcXgoBQAAAAAAgNjxUAoAAAAAAACx46EUAAAAAAAAYkej8zSi0XliCrKppHrtkiVLytqVK1d62Z49e2Rty5YtvaxLly6ydt68eb 1xISFNB4vXry4zFXj0i1btsha1RzTauiomgZaa3vjjTe87IILLoh8XEtIQ/yCbiqproN1zVTz1h9//FHWDh8 PPIaQs5ByHtWnyHVHNQ5u5GudS6U5s2be5lqUu6cfs/WZ14NQlBN3J1zLjs728tU82/n9P3IOr/79u2T eGkYo9 /PHHXta5c2dZ279/fy/7 9//nvQaQiQ7EMP6PFgN/StVquRlmzZtkrXnnHOOl3366aeR1xYipBl/KhzpTaTj1K5dO5kvWrQo6WNnwvVQ933rM5yKz/bhZMI5KSpCrsGaNWtkXr9 /RStJvMU9G9S/Jt1HdT9ZseOHbI2FUNeMhWNzgEAAAAAAJBxeCgFAAAAAACA2PFQCgAAAAAAALHjoRQAAAAAAABix0MpAAAAAAAAxC76iCKgkFLd/q3JVqrWmpg1YsQIL7vrrrtkrZq8sHv3blmrJshYU8bU2qzJYWqqn/XerNdTa65SpYqsVVPJFi9eLGsVayKIukYhk0esWvV6iU4vS8SuXbu8zJo6FzKtMGSyijqG9fdqcki5cuVkbcgUxPvuu8/LHn30UVlbunRpmefn53uZmljqnP68WJMx1YRTNVnTOf3ZKlGihKxVn7fCPhFHTdqz9kHIpD21x6wpNsorr7wi88suuyzyMZTQ6zVq1Cgv  Mf/yhr0zVpL Tzjvi0aNFC5l9//XVSx/38889lfuONN8r89ddf97Lx48cntYZQOTk5Xpabmytrrd8tKPysqdSNGjXysrp166Z7OYC0evVqmZcvX97LrP92OpLxL6UAAAAAAAAQOx5KAQAAAAAAIHY8lAIAAAAAAEDseCgFAAAAAACA2NHoHEVeyZIlvezEE0 UtaqZotU885577vEy1WDZOd081mpMHdKsU7231q1bR/57aw1z5syR cknn xlIQ3JVXN4y4IFCyLXqkbyzjl38OBBL1MN2C1TpkyR YABAyIf438VK1ZM5mXLlvWyYcOGydqHHnrIy6xrOWHCBC8bOHCgrC1VqlTk46rmuGPGjJG1qsF1 /btZe25557rZevXr5e1VtNqteYyZcrI2lmzZnlZr169ZG3jxo1lrnTr1s3L9u7dK2tVw8vq1avLWtWYfeTIkZHXlWrWZ Hjjz/2srVr1yb9eqqpecWKFWWtaohsNWpes2aNl7366quyVg20sJrrWvvulltukXnUY1iDMkKoe7e6Dzmn9666v4a8VjJCvncymXofP/30k6xV3/fWsBJF3d dc2706NEyV/dz67s2XdRn2BqYEPfakHrW59r6XasaSAMFpUKFCjJXAxus34NHMv6lFAAAAAAAAGLHQykAAAAAAADEjodSAAAAAAAAiB0PpQAAAAAAABA7HkoBAAAAAAAgdkzfS6PZs2fLfNGiRTK3plEhmpCpO9aEBDUNISsrS9bu27fPy6ypMGpamkVNH1GToZxzrl69el4Wch62bNki8x49eshcTUZTUwid05N rElBpUuX9jJrQuInn3ziZZ06dZK1Ssj5SWbKnuXFF1 U WWXXeZl1iSskClDIVOv1GRAa0rYww8/7GXW/lfnfMOGDbJWXUtrypeaLOicfs/WdDQ1aa9q1aqydvPmzV5mXQu1huLF9VeuOu95eXmyVp236dOny9pEVatWTeYbN270sjfffDPyca3PXsgebdiwoZfVrl1b1qrzmq6pbdbENIv6XrEmg6Zi0l5Uu3btSstx586dm5bjFnZq3 3cuTMtr6V szinp/o5p/ejNcEvTkzZK7qs 7D1fcj0PWSSr7/ WuZqGrP135ZHMv6lFAAAAAAAAGLHQykAAAAAAADEjodSAAAAAAAAiB0PpQAAAAAAABC7ow5F7O4Z0ogUv81qdrtw4UIva926taxt2rSpl61atSq5hWWIRBvOrlixQuaXXHKJl33xxRdJv7ZqQm01pFXNGLdt2xZ5DSENpFVzX fsZtFKfn6 zPv27etls2bNinxc6z4S0rj0wIEDXmadn2HDhnnZ   /L2uXLl0aeQ3poM6NdV7UdW/cuLGsVU1zN23aJGtVo/ ePXvK2nnz5nlZSBPpVDScPuuss2RuXeOorxeytgYNGsjaH374IanjWtT1LFeuXNLHjUKt3xrQYd1jlWQbF1sDAR5//HEvu/baa2VtyD5Q67X24tSpU2UeQjUft96zGhphNVAPoc5PxYoVZa313Rb1uMkobL9V1Xqtc3LppZd62cSJE1O pv9Qv1t27Ngha9V3cMhvjlRQ692 fXvSx010jw4aNEjmL7/8chKryWzpugYW9d9CVkP/H3/8MW3rKGjJ3EdD/vsCiRkyZIiXWQOwKlWqlO7lFJjD7Sn pRQAAAAAAABix0MpAAAAAAAAxI6HUgAAAAAAAIgdD6UAAAAAAAAQOx5KAQAAAAAAIHZM30PGSXTiQ8getV4jZBJOnz59vGzmzJmyVk2os6aajRo1ysueeeYZWfvdd9952eTJk2Vt/fr1vez444 XtX/7299kriZXbdy4UdZWq1ZN5lFVqVJF5lu2bPGykClZ1vTLuCYFWed88eLFXpaXlydr1dQra5/WqlXLy3755RdzfVGPq8ydO1fm3bt397KcnBxZu3XrVi bM2eOrD3zzDNlribUWdQ 27x5s6xVU2qsyXFqT1p7T02qa9asmawtWbKklxXkBDM1Gc45PR2uRIkSslZ99kIm8ll7acOGDZHXUNhYE5OSnWSYLtZ6E73vFrbfpKVKlZL5Kaec4mXTp09P93IQII7fpJmsevXqXqburSg4yfwGKCr7FJmP6XsAAAAAAADIODyUAgAAAAAAQOx4KAUAAAAAAIDY8VAKAAAAAAAAsdNdV4FCyGqgduDAAS/Lz8 XtSFNYlVTc6tpq9XkVbn11lsjr0s1KGzbtq2sVc2UTz31VFk7e/ZsmQ8ZMiSptakG3c45t23bNi9LRfNmtTbrGqn1tm7dWtaqpuRRffnllzIfPny4l1nnKzs728us66CamlvnNqR5abKDBXJzc2Vtp06dvKxz586yNqShubVedQyrtkyZMl529NFHy1r1nkPWENKQ2zoPap9EYa1TvSfV0Nw550aOHOllt912W Q11KlTR Y///yzlz3xxBOyVp3DkKEIrVq1krXLli3zMtWA3jnn9u/fL/NkpaKhecgeVW6//XaZjxgxwstatGgRfWGFRMjnZNCgQbL22Wefjfx6p512mpfNmDEj8t8XRmPHjvWy66 /Pi2vpRp6g6bmAOLBv5QCAAAAAABA7HgoBQAAAAAAgNjxUAoAAAAAAACx46EUAAAAAAAAYsdDKQAAAAAAAMTuqEOpGHEFAAAAAAAABOBfSgEAAAAAACB2PJQCAAAAAABA7HgoBQAAAAAAgNjxUAoAAAAAAACx46EUAAAAAAAAYsdDKQAAAAAAAMSOh1IAAAAAAACIHQ lAAAAAAAAEDseSgEAAAAAACB2/w /iOYauSvjAwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## MLP/Backpropagation"
      ],
      "metadata": {
        "id": "EQKOS9CF51Sa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def relu(x):\n",
        "    return jnp.maximum(0, x)\n",
        "\n",
        "def softmax(x):\n",
        "    # subtract the max of x for numerical stability\n",
        "    x = x - jnp.max(x)\n",
        "    # compute the exponential of x\n",
        "    exp_x = jnp.exp(x)\n",
        "    # normalize by the sum of exp_x\n",
        "    return exp_x / jnp.sum(exp_x)\n",
        "\n",
        "def mlp_forward(mlp, x):\n",
        "    for i in range(len(mlp) - 1):\n",
        "        weights, biases = mlp[i]\n",
        "        x = jnp.dot(weights, x)\n",
        "        x = jnp.add(biases, x)\n",
        "        x = relu(x)\n",
        "    weights, biases = mlp[-1]\n",
        "    x = jnp.dot(weights, x)\n",
        "    x = jnp.add(biases, x)\n",
        "    x = softmax(x)\n",
        "    return x\n",
        "\n",
        "# Vectorize forward pass so it can handle multiple images per batch.\n",
        "batched_mlp_forward = vmap(mlp_forward, in_axes=(None, 0))\n",
        "\n",
        "def mse(mlp, images, labels):\n",
        "    predictions = batched_mlp_forward(mlp, images)\n",
        "    return jnp.mean(jnp.square(labels - predictions))\n",
        "\n",
        "def update_mlp(mlp, images, labels, lr):\n",
        "    loss, grads = value_and_grad(mse)(mlp, images, labels)\n",
        "    return loss, jax.tree_map(lambda p, g: p - lr * g, mlp, grads)\n",
        "\n",
        "def create_mlp(parent_key, layer_widths):\n",
        "    mlp = []\n",
        "    layer_keys = jax.random.split(parent_key, len(layer_widths))\n",
        "    \n",
        "    for i in range(1, len(layer_widths)):\n",
        "        weight_and_bias_keys = jax.random.split(layer_keys[i], 2)\n",
        "        layer_weights = jax.random.normal(weight_and_bias_keys[0], shape=(layer_widths[i], layer_widths[i-1]))\n",
        "        layer_biases = jax.random.normal(weight_and_bias_keys[1], shape=(layer_widths[i],))\n",
        "        mlp.append((layer_weights, layer_biases))\n",
        "\n",
        "    return mlp\n",
        "\n",
        "def summarize_mlp(mlp):\n",
        "    for i, layer in enumerate(mlp):\n",
        "        weight_summary = ', weights shape: '   str(layer[0].shape)\n",
        "        bias_summary = ', bias shape: '   str(layer[1].shape)\n",
        "        print('Layer '   str(i)   weight_summary   bias_summary)\n",
        "\n",
        "# Set hyperparameters.\n",
        "key = jax.random.PRNGKey(20)\n",
        "mlp = create_mlp(key, [784, 800, 10])\n",
        "epochs = 10\n",
        "batch_size = 32\n",
        "learning_rate = 0.0001\n",
        "\n",
        "# Show summary of model.\n",
        "summarize_mlp(mlp)\n",
        "\n",
        "# Load data.\n",
        "train_loader, test_loader = load_mnist(batch_size)\n",
        "\n",
        "# Train.\n",
        "losses = []\n",
        "for epoch in range(epochs):\n",
        "    cumulative_loss = []\n",
        "    for images, labels in train_loader:\n",
        "        loss, mlp = update_mlp(mlp, images, labels, 0.001)\n",
        "        cumulative_loss.append(loss)\n",
        "    losses.append(sum(cumulative_loss) / len(cumulative_loss))\n",
        "    print('Epoch:', epoch   1, 'Loss:', losses[-1])\n",
        "\n",
        "\n",
        "fig, ax = plt.subplots()\n",
        "ax.plot([epoch for epoch in range(epochs)], losses)\n",
        "ax.set_xlabel('Epoch')\n",
        "ax.set_ylabel('Loss')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "YK7LR4KX_Nf9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 664
        },
        "outputId": "476a88a6-3b4c-4667-9f93-dd23d58e5d58"
      },
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Layer 0, weights shape: (800, 784), bias shape: (800,)\n",
            "Layer 1, weights shape: (10, 800), bias shape: (10,)\n",
            "Epoch: 1 Loss: 0.1801504\n",
            "Epoch: 2 Loss: 0.18004031\n",
            "Epoch: 3 Loss: 0.17992896\n",
            "Epoch: 4 Loss: 0.1798279\n",
            "Epoch: 5 Loss: 0.1797907\n",
            "Epoch: 6 Loss: 0.17972516\n",
            "Epoch: 7 Loss: 0.1796904\n",
            "Epoch: 8 Loss: 0.1795577\n",
            "Epoch: 9 Loss: 0.179457\n",
            "Epoch: 10 Loss: 0.17937404\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlEAAAGwCAYAAACJjDBkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABaR0lEQVR4nO3deVxVdeL/8de57DsomyiKqLnmLojV2EKamY1mmmWJ2qaZI/JzUqcx69tiZTWOS9piZJlltmeLC7apKARh7pq5ooCogIICcs/vD6c7Q2opgofl/Xw87uMh537uOe D1X13zud rmGapomIiIiIXBSb1QFEREREaiKVKBEREZEKUIkSERERqQCVKBEREZEKUIkSERERqQCVKBEREZEKUIkSERERqQBnqwPUZna7nYMHD Lj44NhGFbHERERkQtgmibHjx8nLCwMm 3815tUoqrQwYMHCQ8PtzqGiIiIVMD /ftp1KjReZ9XiapCPj4 wJm/BF9fX4vTiIiIyIUoKCggPDzc8T5 PipRVei3W3i vr4qUSIiIjXMn03F0cRyERERkQpQiRIRERGpAJUoERERkQpQiRIRERGpAJUoERERkQpQiRIRERGpAJUoERERkQpQiRIRERGpAJUoERERkQpQiRIRERGpAJUoERERkQpQiRIRERGpAJWoGuhUaRnf7ThsdQwREZE6TSWqhikts/PwonSGJ6awaP0 q OIiIjUWSpRNYyTYRDm74Fpwj8 3sjrP/xqdSQREZE6SSWqhrHZDJ64tS2jejYD4KkvtvLvlTsxTdPiZCIiInWLSlQNZBgGE29qyYReVwDwr5U7mPbVNhUpERGRy0glqoYyDIOHr2/BY7e0AeDV73/ln59swm5XkRIREbkcVKJquJFXN W5gVdiGPDO n1MWLKB02V2q2OJiIjUeipRtcAd3Roz446OONkMPvopk4cX/UTx6TKrY4mIiNRqKlG1xF87NmTu0M64Otn4enMWD7yVxskSFSkREZGqohJVi/RqG8r84V3xcHHiux2HGZ6Ywoni01bHEhERqZVUomqZa1oE8fa9Ufi4ObN 91GGvr6evKISq2OJiIjUOipRtVDXiHosur87AZ4ubNifx5BX13H4eLHVsURERGoVlaha6spGfix MIYgHze2ZR3njleSOZh30upYIiIitYZKVC12RYgPSx6MoaG/B7/mFjJoXjJ7jxRaHUtERKRWUImq5SICvXh/VAxNA73IzDvJoHnJ7Mw bnUsERGRGk8lqg5o6O/B4ge70zLEh5zjxdzx6jo2ZeZbHUtERKRGU4mqI4J93Hnvge60b TH0cIS7nx1HWl7j1odS0REpMZSiapDArxceee aKIi6nG8 DR3v57Cml9yrY4lIiJSI6lE1TE 7i4sGBnFNS0COVlaxog3U1m5JdvqWCIiIjWOSlQd5OHqxOtxXenVJoSS03ZGLUzj8w0HrY4lIiJSo1heoubMmUNERATu7u5ER0eTkpJy3rGbN29m4MCBREREYBgGM2bMOGtMWVkZU6ZMoWnTpnh4eNCsWTOefPJJTNN0jDFNk8cee4wGDRrg4eFBbGwsO3fuLLefp59 mh49euDp6Ym/v39lnW614ebsxJyhnenfMYzTdpNx7/3E 6n7rY4lIiJSY1haohYvXkxCQgJTp04lPT2dDh060Lt3b3Jycs45vqioiMjISJ599llCQ0PPOea5555j7ty5zJ49m61bt/Lcc8/x/PPPM2vWLMeY559/npkzZzJv3jzWr1 Pl5cXvXv35tSpU44xJSUlDBo0iNGjR1fuSVcjLk42XhrckTujGmM34ZEPfyZxzW6rY4mIiNQIhvm/l2gus joaLp168bs2bMBsNvthIeHM3bsWCZNmvSHr42IiCA Pp74 Phy22 55RZCQkKYP3  Y9vAgQPx8PBg4cKFmKZJWFgY/ ///T8mTJgAQH5 PiEhIbz55psMGTKk3P7efPNN4uPjycvLu jzKygowM/Pj/z8fHx9fS/69ZeLaZo8/cVWXl99pkD9vXdLxlzX3OJUIiIi1rjQ92/LrkSVlJSQlpZGbGzsf8PYbMTGxpKcnFzh/fbo0YOkpCR27NgBwIYNG1i9ejV9 vQBYPfu3WRlZZU7rp fH9HR0Zd0XIDi4mIKCgrKPWoCwzB4tG9rxt3QAoDpy7bz/NfbsLBfi4iIVHvOVh04NzeXsrIyQkJCym0PCQlh27ZtFd7vpEmTKCgooFWrVjg5OVFWVsbTTz/N0KFDAcjKynIc5/fH/e25ipo2bRpPPPHEJe3DKoZhMP7GK/Byc KZL7fx8re7KCop47Fb2mCzGVbHExERqXYsn1he2d5//33eeecdFi1aRHp6OgsWLOCFF15gwYIFVX7syZMnk5 f73js31/zJmo/8JdmPNW/HYYBb67dw8QPf6bMritSIiIiv2fZlajAwECcnJzIzi6/RlF2dvZ5J41fiL///e9MmjTJMbfpyiuvZO/evUybNo24uDjHvrOzs2nQoEG543bs2LHCxwVwc3PDzc3tkvZRHdzdvQmerk5MWLKBJWkHKCot41 DO LqXOs6t4iISIVZ9q7o6upKly5dSEpKcmyz2 0kJSURExNT4f0WFRVhs5U/LScnJ x2OwBNmzYlNDS03HELCgpYv379JR23trmtcyNeHtoZFyeDL34 xOiFaZwqLbM6loiISLVh2ZUogISEBOLi4ujatStRUVHMmDGDwsJCRowYAcCwYcNo2LAh06ZNA85MRt yZYvjz5mZmWRkZODt7U3z5mc TdavXz efvppGjduTNu2bfnpp8636aWXGDlyJHBm7k98fDxPPfUULVq0oGnTpkyZMoWwsDD69 /vyLZv3z6OHj3Kvn37KCsrIyMjA4DmzZvj7e19mX5D1rqpXQNeG bEg2 nkbQth5FvpvLasK54uVn6j42IiEj1YFps1qxZZuPGjU1XV1czKirKXLduneO5nj17mnFxcY6fd /ebQJnPXr27OkYU1BQYI4bN85s3Lix6e7ubkZGRpqPPvqoWVxc7Bhjt9vNKVOmmCEhIaabm5t5ww03mNu3by XKy4u7pzH uabby743PLz803AzM/Pv jfS3WSvCvXbDPlK7PJxKXmgDmrzbyiEqsjiYiIVJkLff 2dJ2o2q6mrBN1IX7ad4zhianknyylbZgvb42Mor53zZ//JSIi8nvVfp0oqVk6NQ7gvQe6E jtyuaDBdzx6jqyC079 QtFRERqKZUouWCtG/iy MEYGvi580vOCQbNS2b/0SKrY4mIiFhCJUouSrMgb95/MIbG9TzZd7SIQfOS2XX4hNWxRERELjuVKLlo4fU8WTIqhubB3mQVnOKOV5LZeqhmfMWNiIhIZVGJkgoJ8XVn8QPdaRvmS 6JEoa8uo6M/XlWxxIREblsVKKkwup7u7Ho/u50aRJA/slShr62jnW/HrE6loiIyGWhEiWXxM/DhbdGRtGjWX0KS8qIeyOFb7bnWB1LRESkyqlEySXzcnPmjeHduKFVMMWn7Tzw1o98tfGQ1bFERESqlEqUVAp3Fyfm3dOFvu0bUFpmMmZROh lH7A6loiISJVRiZJK4 JkY aQTgzu2gi7CQnvb2Dhur1WxxIREakSKlFSqZxsBs/e1p7hPSIA Ocnm3jlu13WhhIREakCKlFS6Ww2g6n92jDmumYATPtqGy t2IG plFERGoTlSipEoZh8Pferfh775YAzEzayVNfbFWREhGRWkMlSqrUmOua83i/NgDMX72bf3y8iTK7ipSIiNR8KlFS5YZf1ZTnB7bHZsC7Kfv4f 9ncLrMbnUsERGRS6ISJZfF4G7hzLyzE842g08yDvLQO kUny6zOpaIiEiFqUTJZXNL zBeuacLrs42lm/J5r4FP3KyREVKRERqJpUouaxuaB1C4vBueLo68cPOXIa9sZ7jp0qtjiUiInLRVKLksruqeSBv3xuFj7szqXuOcffr68krKrE6loiIyEVRiRJLdGlSj3fv706ApwsbDuQz5NV15J4otjqWiIjIBVOJEsu0a jHew/EEOjtxras4wx5dR3ZBaesjiUiInJBVKLEUi1DfXj/we408HPnl5wTDH4lmQPHiqyOJSIi8qdUosRykUHevP9gDOH1PNh7pIg7XlnH3iOFVscSERH5QypRUi2E1/Pk/QdjiAz0IjPvJIPmJfNLznGrY4mIiJyXSpRUGw38PFj8YAwtQ3zIOV7MHa sY8vBAqtjiYiInJNKlFQrQT5uvPtAd9o19OVIYQl3vraODfvzrI4lIiJyFpUoqXbqebnyzn3d6dTYn/yTpQx9fT2pe45aHUtERKQclSiplvw8XHj73mi6R9bjRPFphs1PYe0vuVbHEhERcVCJkmrL282ZxOFR/OWKIE6WljH8zVS 2ZZjdSwRERFAJUqqOQ9XJ14b1oUb24RQctrOA2//yNebsqyOJSIiohIl1Z bsxMvD 1M3/YNKC0zGbMonU8zMq2OJSIidZxKlNQILk42Zg7pxMDOjSizm8QvzuD91P1WxxIRkTqsWpSoOXPmEBERgbu7O9HR0aSkpJx37ObNmxk4cCAREREYhsGMGTPOGlNWVsaUKVNo2rQpHh4eNGvWjCeffBLTNB1jTNPkscceo0GDBnh4eBAbG8vOnTvL7efo0aMMHToUX19f/P39uffeezlx4kSlnbdcHCebwfTb2zM0ujGmCY98 DNvJe xOpaIiNRRlpeoxYsXk5CQwNSpU0lPT6dDhw707t2bnJxzTyAuKioiMjKSZ599ltDQ0HOOee6555g7dy6zZ89m69atPPfcczz//PPMmjXLMeb5559n5syZzJs3j/Xr1 Pl5UXv3r05deq/X4A7dOhQNm/ezIoVK1i6dCnff/89DzzwQOX AuSi2GwGT/Vvx8irmgLw2KebefX7XRanEhGROsm0WFRUlDlmzBjHz2VlZWZYWJg5bdq0P31tkyZNzH/9619nbe/bt685cuTIcttuu 02c jQoaZpmqbdbjdDQ0PN6dOnO57Py8sz3dzczHfffdc0TdPcsmWLCZipqamOMV999ZVpGIaZmZl5QeeWn59vAmZ fv4FjZcLZ7fbzelfbzObTFxqNpm41JyxYodpt9utjiUiIrXAhb5/W3olqqSkhLS0NGJjYx3bbDYbsbGxJCcnV3i/PXr0ICkpiR07dgCwYcMGVq9eTZ8 fQDYvXs3WVlZ5Y7r5 dHdHS047jJycn4 /vTtWtXx5jY2FhsNhvr168/53GLi4spKCgo95CqYRgGE3q3ZEKvKwD418odPL9se7lbtiIiIlXJ2cqD5 bmUlZWRkhISLntISEhbNu2rcL7nTRpEgUFBbRq1QonJyfKysp4 umnGTp0KABZWVmO4/z uL89l5WVRXBwcLnnnZ2dqVevnmPM702bNo0nnniiwrnl4j18fQvcXZx46outzP12FydLypjarw2GYVgdTUREajnL50RVhffff5933nmHRYsWkZ6ezoIFC3jhhRdYsGBBlR538uTJ5OfnOx779 vTY5fDfddE8lT/dgC8uXYP//h4I3a7rkiJiEjVsvRKVGBgIE5OTmRnZ5fbnp2dfd5J4xfi73//O5MmTWLIkCEAXHnllezdu5dp06YRFxfn2Hd2djYNGjQod9yOHTsCEBoaetbk9tOnT3P06NHzZnNzc8PNza3CuaXi7u7eBHcXJx75YAPvpuznVKmd6be3x9mpVv5/goiIVAOWvsO4urrSpUsXkpKSHNvsdjtJSUnExMRUeL9FRUXYbOVPzcnJCbvdDkDTpk0JDQ0td9yCggLWr1/vOG5MTAx5eXmkpaU5xqxatQq73U50dHSFs0nVub1LI/49pBNONoOPf8pk7Ls/UXLabnUsERGppSy9EgWQkJBAXFwcXbt2JSoqihkzZlBYWMiIESMAGDZsGA0bNmTatGnAmcnoW7Zscfw5MzOTjIwMvL29ad68OQD9 vXj6aefpnHjxrRt25affvqJl156iZEjRwJnJiXHx8fz1FNP0aJFC5o2bcqUKVMICwujf// ALRu3ZqbbrqJ   /n3nz5lFaWsrDDz/MkCFDCAsLu8y/JblQ/TqE4eZs4 FFP/HVpixKFqYxZ2hn3F2crI4mIiK1zeX5sOAfmzVrltm4cWPT1dXVjIqKMtetW d4rmfPnmZcXJzj5927d5vAWY ePXs6xhQUFJjjxo0zGzdubLq7u5uRkZHmo48 ahYXFzvG2O12c8qUKWZISIjp5uZm3nDDDeb27dvL5Tpy5Ih55513mt7e3qavr685YsQI8/jx4xd8XlriwDrfbs8xr3j0S7PJxKXm0NfWmYXFpVZHEhGRGuJC378N09RnwqtKQUEBfn5 5Ofn4 vra3WcOid51xHuXZBKUUkZURH1eGNEN7zdLL/4KiIi1dyFvn9r1q3UWjHN6vP2vdH4uDmTsucod7  nvyiUqtjiYhILaESJbValyYBLLq/O/6eLmTsz PO19Zx5ESx1bFERKQWUImSWu/KRn6890B3Ar1d2XKogCGvriOn4NSfv1BEROQPqERJndAq1JfFD8YQ6uvOzpwTDH4lmcy8k1bHEhGRGkwlSuqMZkHevP9gDI0CPNhzpIjB85LZd6TI6lgiIlJDqURJndK4vifvPxhD00AvMvNOMuiVtfySc8LqWCIiUgOpREmdE bvweIHu3NFiDfZBcUMeTWZbVkFVscSEZEaRiVK6qRgH3feeyCGNg18yT1RwpBX17HxQL7VsUREpAZRiZI6q56XK /e352O4f7kFZVy12vrSNt71OpYIiJSQ6hESZ3m5 nCwvuiiWpaj PFp7lnfgprd VaHUtERGoAlSip87zdnFkwIoprWgRSVFLGiMRUvt2eY3UsERGp5lSiRAAPVydeG9aVG1oFU3zazv1v/ciyzVlWxxIRkWpMJUrkP9xdnJh7dxf6XtmA0jKTh95J5/MNB62OJSIi1ZRKlMj/cHW28e8hHbmtU0PK7Cbj3vuJJT/utzqWiIhUQypRIr/j7GTjhUEduDOqMXYT/v7Bz7y9bq/VsUREpJpRiRI5B5vN4JkB7RjeIwKAKZ9s4vUffrU2lIiIVCsqUSLnYRgGU/u1YfS1zQB46outzEraaXEqERGpLlSiRP6AYRg80rslCTdeAcCLK3Ywfdk2TNO0OJmIiFhNJUrkTxiGwd9uaME/bm4FwJxvdvHk0q0qUiIidZxKlMgFeuAvzXjyr20BeGPNbh79ZBN2u4qUiEhdpRIlchHuiYng dvbYxiwaP0 JnywgdNldqtjiYiIBVSiRC7S4K7hzLijI042g4/SMxm3OINSFSkRkTpHJUqkAv7asSFz7uqMi5PBFz8fYvTCdE6VllkdS0RELiOVKJEKuqldKK8O64qbs42VW7OJeyOF/KJSq2OJiMhlohIlcgmuaxlM4vBueLs5s373UQbOW8v o0VWxxIRkctAJUrkEvVoHsiSUTGE rrzS84JBry8lp8P5FkdS0REqphKlEglaN3Al4/H9KBVqA 5J4q545V1JG3NtjqWiIhUIZUokUrSwM DJaNiuKZFICdLy7j/rR/1xcUiIrWYSpRIJfJxd GN4d0Y3LURdvPMFxdP 2qrFuUUEamFVKJEKpmLk43nBrZ3fN/eK9/9yt/e 0lLIIiI1DIqUSJV4Lfv23tpcAdcnAyW/nyIe av51hhidXRRESkkqhEiVSh2zo3YsGIKHzcnUndc4yBc9ey74iWQBARqQ1UokSqWI/mgXw4ugdhfu78mlvIgJfXkLE/z pYIiJyiSwvUXPmzCEiIgJ3d3eio6NJSUk579jNmzczcOBAIiIiMAyDGTNmnDXmt d /xgzZoxjzK5duxgwYABBQUH4 voyePBgsrPLfxw9PT2dG2 8EX9/f rXr88DDzzAiRMnKu28pW65IsSHj8dcRdswX44UljDk1WSWb86yOpaIiFwCS0vU4sWLSUhIYOrUqaSnp9OhQwd69 5NTk7OOccXFRURGRnJs88 S2ho6DnHpKamcujQIcdjxYoVAAwaNAiAwsJCevXqhWEYrFq1ijVr1lBSUkK/fv2w2898iezBgweJjY2lefPmrF /nq //prNmzczfPjwyv8lSJ0R4uvO4gdjuLZlEKdK7Ty4MI031 y2OpaIiFSQYZqmZZ 9jo6Oplu3bsyePRsAu91OeHg4Y8eOZdKkSX/42oiICOLj44mPj//DcfHx8SxdupSdO3diGAbLly nT58 HDt2DF9fXwDy8/MJCAhg fLlxMbG8uqrrzJlyhQOHTqEzXamZ27cuJH27duzc dOmjdvfs5jFRcXU1xc7Pi5oKCA8PBw8vPzHccSOV1mZ8qnm3k3ZR8A913dlH/c3BqbzbA4mYiIwJn3bz8/vz99/7bsSlRJSQlpaWnExsb N4zNRmxsLMnJyZV2jIULFzJy5EgM48wbVHFxMYZh4Obm5hjn7u6OzWZj9erVjjGurq6OAgXg4eEB4BhzLtOmTcPPz8/xCA8Pr5TzkNrF2cnGMwPa8chNLQF4ffVuxixK1xIIIiI1jGUlKjc3l7KyMkJCQsptDwkJISurcuaKfPLJJ Tl5ZW7Dde9e3e8vLyYOHEiRUVFFBYWMmHCBMrKyjh06BAA119/PVlZWUyfPp2SkhKOHTvmuDL225hzmTx5Mvn5 Y7H/v37K U8pPYxDIOHrm3Ov4d0xNXJxlebsrjrtXUc1RIIIiI1huUTy6vS/Pnz6dOnD2FhYY5tQUFBLFmyhM8//xxvb2/8/PzIy8ujc fOjitPbdu2ZcGCBbz44ot4enoSGhpK06ZNCQkJKXd16vfc3Nzw9fUt9xD5I3/t2JC37o3C192Z9H153PbyGvbkFlodS0RELoBlJSowMBAnJ6ezPhWXnZ193knjF2Pv3r2sXLmS  6776znevXqxa5du8jJySE3N5e3336bzMxMIiMjHWPuuususrKyyMzM5MiRIzz  OMcPny43BiRytA9sj4fPdSDRgEe7DlSxICX15C295jVsURE5E9YVqJcXV3p0qULSUlJjm12u52kpCRiYmIuef JiYkEBwfTt2/f844JDAzE39 fVatWkZOTw6233nrWmJCQELy9vVm8eDHu7u7ceOONl5xN5PeaB/vw0UM9uLKhH8eKSrnrtXV8tfH8t45FRMR6lt7OS0hI4LXXXmPBggVs3bqV0aNHU1hYyIgRIwAYNmwYkydPdowvKSkhIyODjIwMSkpKyMzMJCMjg19  aXcfu12O4mJicTFxeHs7HzWcRMTE1m3bh27du1i4cKFDBo0iPHjx9OyZUvHmNmzZ5Oens6OHTuYM2cODz/8MNOmTcPf379qfhlS5wX7uLP4we7c0CqY4tN2HlqUzvzVWgJBRKTaMi02a9Yss3Hjxqarq6sZFRVlrlu3zvFcz549zbi4OMfPu3fvNoGzHj179iy3z2XLlpmAuX379nMec LEiWZISIjp4uJitmjRwnzxxRdNu91ebsw999xj1qtXz3R1dTXbt29vvvXWWxd9bvn5 SZg5ufnX/Rrpe4qPV1m/vPjjWaTiUvNJhOXmlM/3WSeLrP/ QtFRKRSXOj7t6XrRNV2F7rOhMjvmabJaz/8yjNfbgPgxjYhzBzSCQ9XJ4uTiYjUftV nSgROT/DMHjgL82YfVcnXJ1trNiSzZDX1pF7ovjPXywiIpeFSpRINXZL zDeuS8af08XNuzP47aX17LrsL7DUUSkOlCJEqnmukXU48PRPWhcz5N9R4sYOHctqXuOWh1LRKTOU4kSqQGaBXnz0UM96BjuT15RKUNfX8/Snw9aHUtEpE5TiRKpIQK93Xj3/u70ahNCyWk7Dy/6iVe 24U GyIiYg2VKJEaxMPVibl3d2F4jwgApn21jcc 3czpMru1wURE6iCVKJEaxslm8PitbZlySxsMA95et5cH306jqOS01dFEROoUlSiRGureq5syd2hn3JxtJG3L4Y5X1pFz/JTVsURE6gyVKJEa7KZ2DVh0f3fqebmyMTOfAXPW8kvOcatjiYjUCSpRIjVclyYBfDS6BxH1PcnMO8ltL69l3a9HrI4lIlLrqUSJ1AIRgV589NBVdG7sT8Gp0wybn8KnGZlWxxIRqdVUokRqiXperiy6vzt92oVSUmZn3HsZzPnmFy2BICJSRVSiRGoRdxcn5tzVmfuvaQrA9GXb cfHG7UEgohIFVCJEqllbDaDR/u24fF Z5ZAeDdlP/e99SMnirUEgohIZVKJEqmlhl/VlFfu7oK7i41vtx/mjleSyS7QEggiIpVFJUqkFuvVNpT3Hogh0NuVzQcLGDBnDduztASCiEhlUIkSqeU6hvvz0eiriAzy4mD KW6fu5a1v RaHUtEpMZTiRKpAxrX9 Sj0T3oFhHA8eLTxCWm8FH6AatjiYjUaCpRInWEv6crb98bzS3tG1BaZpLw/gZmJu3UEggiIhWkEiVSh7i7ODFzSCdG9WwGwEsrdvDIBz9TqiUQREQumkqUSB1jsxlM6tOKp/q3w2bAkrQDjHwzleOnSq2OJiJSo6hEidRRd3dvwutxXfF0deKHnbkMmpfMofyTVscSEakxVKJE6rDrW4Ww IEYgnzc2JZ1nAFz1vLVxkOcLCmzOpqISLVnmJpVWmUKCgrw8/MjPz8fX19fq OInNeBY0UMT0zll5wTALi72Lj2imBuahfK9a2D8XV3sTihiMjlc6Hv3ypRVUglSmqS/KJSXv72F77YeIgDx/57W8/FyeCq5oHc1DaUG9uEUN/bzcKUIiJVTyWqGlCJkprINE02Hyxg2eYsvtqU5bg6BWAzoFtEPfq0C6V3u1Aa HlYmFREpGqoRFUDKlFSG/ySc5xlm7P5elMWGzPzyz3XIdyfPu1CualtKBGBXhYlFBGpXCpR1YBKlNQ2 48WsWxzFss2Z/Hj3mP87389WoX60LttKDe1C6VVqA GYVgXVETkEqhEVQMqUVKb5Rw/xfLN2SzbnEXyriOctv/3PyUR9T3p/Z8rVB0a WOzqVCJSM2hElUNqERJXZFXVELS1hy 2pTF9zsPU3L6vyugh/q6c1O7UHq3DaVbRADOTlpZRUSqN5WoakAlSuqiwuLTfLv9MF9tOsQ323Io/J81p p5uXJj6xBuujKUHs3q4 bsZGFSEZFzU4mqBlSipK47VVrGml9y XpTFiu2ZpNX9N vlvFxc b61sHc1DaUni2D8HR1tjCpiMh/Xej7t XX1efMmUNERATu7u5ER0eTkpJy3rGbN29m4MCBREREYBgGM2bMOGvMb8/9/jFmzBjHmF27djFgwACCgoLw9fVl8ODBZGdnl9vPjh07 Otf/0pgYCC vr5cffXVfPPNN5V23iJ1gbuLEze0DmH6oA6kPhrLO/dFc0/3JgT7uHG8 DSfZhxk9DvpdH5yBQ  /SMf/3SA/JP6Dj8RqRksLVGLFy8mISGBqVOnkp6eTocOHejduzc5OTnnHF9UVERkZCTPPvssoaGh5xyTmprKoUOHHI8VK1YAMGjQIAAKCwvp1asXhmGwatUq1qxZQ0lJCf369cNu/ 88jltuuYXTp0 zatUq0tLS6NChA7fccgtZWVmV/FsQqRtcnGxc1TyQJ/u3Y93kG/hwdAz3X9OURgEenCq1s2xzNuMXb6DrUyuIeyOFd1P2kXui2OrYIiLnZentvOjoaLp168bs2bMBsNvthIeHM3bsWCZNmvSHr42IiCA Pp74 Pg/HBcfH8/SpUvZuXMnhmGwfPly vTpw7FjxxyX6PLz8wkICGD58uXExsaSm5tLUFAQ33//Pddccw0Ax48fx9fXlxUrVhAbG3tB56fbeSJ/zjRNthwq4OtNWXy9KYudv1vcs tvi3u2DSXMX4t7ikjVu9D3b8smIZSUlJCWlsbkyZMd22w2G7GxsSQnJ1faMRYuXEhCQoJjzZri4mIMw8DN7b9fXeHu7o7NZmP16tXExsZSv359WrZsyVtvvUXnzp1xc3PjlVdeITg4mC5dupz3eMXFxRQX//f/nAsKCirlPERqM8MwaBvmR9swP/5fr5b8knOCZZuzHIt7puw Ssruozzx RY6NPJzLJ0QGeRtdXQRqeMsK1G5ubmUlZUREhJSbntISAjbtm2rlGN88skn5OXlMXz4cMe27t274 XlxcSJE3nmmWcwTZNJkyZRVlbGoUOHgDP/UV 5ciX9 /fHx8cHm81GcHAwX3/9NQEBAec93rRp03jiiScqJbtIXdU82Jvmwc0Zc11zDhwrYtnmbJZtyiJ171E2HMhnw4F8nv96Oy1DfByFqnUDLe4pIpef5RPLq9L8 fPp06cPYWFhjm1BQUEsWbKEzz//HG9vb/z8/MjLy6Nz587YbGd HaZpMmbMGIKDg/nhhx9ISUmhf// 9OvXz1G0zmXy5Mnk5 c7Hvv376/ycxSpzRoFeHLv1U15f1QM6/9xA08PaMc1LQJxthlszz7OzKSd3DzzB6863VumfbmV9H3HsNv1gWMRuTwsuxIVGBiIk5PTWZ Ky87OPu k8Yuxd 9eVq5cyUcffXTWc7169WLXrl3k5ubi7OyMv78/oaGhREZGArBq1SqWLl1abt7Uyy /zIoVK1iwYMF552u5ubmVu00oIpUn2MedodFNGBrdhPyiUlZuzebrzVl8v Mwe48U8cr3v/LK978S6utO77Yh3BndmFahmosoIlXHsitRrq6udOnShaSkJMc2u91OUlISMTExl7z/xMREgoOD6du373nHBAYG4u/vz6pVq8jJyeHWW28FznwKEHBcmfqNzWYr9wk EbGGn6cLA7s04rVhXUmfciMvD 1Mvw5heLk6kVVwigXJe7n53z8w aOfOXxcn/ATkaph6ep2CQkJxMXF0bVrV6KiopgxYwaFhYWMGDECgGHDhtGwYUOmTZsGnJkovmXLFsefMzMzycjIwNvbm bNmzv2a7fbSUxMJC4uDmfns08xMTGR1q1bExQURHJyMuPGjWP8 PG0bNkSgJiYGAICAoiLi Oxxx7Dw8OD1157jd27d/9hKRORy8/LzZmbr2zAzVc24FRpGWt35fJ 6gG 3pzFuyn7 XzDIR66rhkjr2qKu4tWSBeRSmRWwL59 8z9 /c7fl6/fr05btw485VXXrnofc2aNcts3Lix6erqakZFRZnr1q1zPNezZ08zLi7O8fPu3btN4KxHz549y 1z2bJlJmBu3779nMecOHGiGRISYrq4uJgtWrQwX3zxRdNut5cbk5qaavbq1cusV6 e6ePjY3bv3t388ssvL rc8vPzTcDMz8 /qNeJyKVL2X3E7DfrB7PJxKVmk4lLzR7TkszPMjLP nddROT3LvT9u0LrRF1zzTU88MAD3HPPPWRlZdGyZUvatm3Lzp07GTt2LI899lhl9rwaS tEiVjLbjf5JCOT57/eTlbBKQC6NAlgyi1t6Bjub204Eam2qvRrXzZt2kRUVBQA77//Pu3atWPt2rW88847vPnmmxUKLCJS2Ww2g9s6N2LVhJ7Ex7bAw8WJtL3H6D9nDfHv/cTBvJNWRxSRGqxCJaq0tNTxKbSVK1c6JmS3atXqD5cAEBGxgqerM/GxV/DNhGu5rXNDAD7JOMj1L37LS8u3U1h82uKEIlITVahEtW3blnnz5vHDDz wYsUKbrrpJgAOHjxI/fr1KzWgiEhlCfVz56XBHfns4avoFhHAqVI7M1f9wnUvfMuSH/drjSkRuSgVKlHPPfccr7zyCtdeey133nknHTp0AOCzzz5z3OYTEamu2jfy5/0HY3h5aGfC63mQc7yYv3/wM7fOWc36X49YHU9EaogKfwFxWVkZBQUF5b4GZc ePXh6ehIcHFxpAWsyTSwXqf5OlZaxYO0eZq/6heP/ua13U9tQJt/ciib1vSxOJyJWuND37wqVqJMnT2KaJp6ensCZ1cE//vhjWrduTe/evSueupZRiRKpOXJPFPOvFTt4N2UfdhNcnWwMvyqCh69vjq 7i9XxROQyqtIS1atXL2677TZGjRpFXl4erVq1wsXFhdzcXF566SVGjx59SeFrC5UokZpne9ZxnvpiCz/szAWgnpcr42 8gju7hePsVKu/blRE/qNKlzhIT0/nmmuuAeCDDz4gJCSEvXv38tZbbzFz5syKJRYRqQZahvrw1sgoEod3o1mQF0cLS5jyySZunvkD3 04bHU8EalGKlSiioqK8PHxAWD58uXcdttt2Gw2unfvzt69eys1oIjI5WYYBte1Cubr L/wxK1t8fd0YUf2CeLeSGF4Ygq/5By3OqKIVAMVKlHNmzfnk08 Yf/ /SxbtoxevXoBkJOTo9tWIlJruDjZiOsRwXcTruPeq5vibDP4dvthes/4gcc 3cTRwhKrI4qIhSpUoh577DEmTJhAREQEUVFRxMTEAGeuSnXq1KlSA4qIWM3P04Upt7RhRUJPbmwTQpnd5K3kvfSc/g2v//ArJaftVkcUEQtUeImDrKwsDh06RIcOHbDZznSxlJQUfH19adWqVaWGrKk0sVykdlr7Sy5PfrGVrYcKAIio78nkm1vTq00IhmFYnE5ELlWVfjrvfx04cACARo0aXcpuaiWVKJHaq8xu8kHafqYv20HuiWIAukfWY8otbWgb5mdxOhG5FFX66Ty73c7//d//4efnR5MmTWjSpAn /v48 eST2O26rC0itZ TzeCObo359u/XMua6Zrg621j361FumbWaRz7YQE7BKasjikgVq9CVqMmTJzN//nyeeOIJrrrqKgBWr17N448/zv3338/TTz9d6UFrIl2JEqk7Dhwr4rmvt/P5hoMAeLo68dC1zbjvmkjcXZwsTiciF6NKb eFhYUxb948br311nLbP/30Ux566CEyMzMvPnEtpBIlUvek7T3Gk0u3kLE/D4CG/h48clNLbu0QpvlSIjVEld7OO3r06Dknj7dq1YqjR49WZJciIrVClyYBfPxQD/49pCNhfu5k5p1k3HsZ3DZ3Len7jlkdT0QqUYVKVIcOHZg9e/ZZ22fPnk379u0vOZSISE1mGAZ/7diQVROuZUKvK/B0deKnfXnc9vJa/vbuT2TmnbQ6oohUggrdzvvuu /o27cvjRs3dqwRlZyczP79 /nyyy8dXwlT1 l2nogA5BSc4oXl21mSdgDTBDdnG/dfE8noa5vh5eZsdTwR Z0qvZ3Xs2dPduzYwYABA8jLyyMvL4/bbruNzZs38/bbb1c4tIhIbRTs687zt3fg84evpntkPYpP25n9zS9c 8K3vJ 6nzL7Ja00IyIWueR1ov7Xhg0b6Ny5M2VlZZW1yxpNV6JE5PdM02T5lmye XIre48UAdCmgS9TbmlDTLP6FqcTEajiK1EiIlIxhmHQu20oK8b35J99W Pj7syWQwXc do6HnjrR/bkFlodUUQukEqUiIgFXJ1t3HdNJN/9/TqGxTTByWawfEs2N/7rO55auoX8k6VWRxSRP6ESJSJioXpervzfX9vx9bhruLZlEKVlJq v3s21079hwdo9lJbpWyBEqquLmhN12223/eHzeXl5fPfdd5oT9R aEyUiF u7HYd5aukWduacAKC lyvXtgwmtnUw11wRhLc zSdS5apkxfIRI0Zc0LjExMQL3WWtphIlIhVxuszOu6n7mbFiB0cKSxzbXZ1sREfWI7Z1CNe3Cia8nqeFKUVqryr92he5MCpRInIpSsvspO45StLWHJK2ZrPnP5/m 03LEB9uaB3MDa1D6Bjuj5NNXysjUhlUoqoBlSgRqUy7Dp8gaWs2K7fmkLb3WLn1pep7uXJdq2BuaKXbfiKXSiWqGlCJEpGqkldUwrfbD7Nyazbf7TjM8VOnHc/9722/G1oH0yhAt/1ELoZKVDWgEiUil8Of3fZrFerD9a1020/kQqlEVQMqUSJyuZmmya7DhSRtzSZpWw4/7jnK/36rzG 3/WJbB3N1C932EzkXlahqQCVKRKym234iF69Gfe3LnDlziIiIwN3dnejoaFJSUs47dvPmzQwcOJCIiAgMw2DGjBlnjfntud8/xowZ4xiza9cuBgwYQFBQEL6 vgwePJjs7GzH899   0592EYBqmpqZV6/iIiVcXf05X nRoy 67OpE 5kUX3RXPv1U2JqO9JSZmdH3bmMvWzzVz93DfcNON7pi/bRvq Y/pSZJELYHmJWrx4MQkJCUydOpX09HQ6dOhA7969ycnJOef4oqIiIiMjefbZZwkNDT3nmNTUVA4dOuR4rFixAoBBgwYBUFhYSK9evTAMg1WrVrFmzRpKSkro168fdvuZ1YF79OhRbh HDh3ivvvuo2nTpnTt2rUKfhMiIlXLxclGj aBTLmlDd9MuJaVCT2Z3KcVURH1sBmwLes4c77ZxW0vryXq6ZVMWLKBrzcdorD49J/vXKQOsvx2XnR0NN26dWP27NkA2O12wsPDGTt2LJMmTfrD10ZERBAfH098fPwfjouPj2fp0qXs3LkTwzBYvnw5ffr04dixY47LdPn5 QQEBLB8 XJiY2PP2kdpaSkNGzZk7NixTJky5YLOTbfzRKSmOFZYwnc7zn/br3uz sS2Dub6VrrtJ7Xfhb5/WzqjsKSkhLS0NCZPnuzYZrPZiI2NJTk5udKOsXDhQhISEjCMM59IKS4uxjAM3NzcHOPc3d2x2WysXr36nCXqs88 48iRI3 4antxcTHFxcWOnwsKCirlHEREqlqA15nbfv07NTzzab/dR1m5NYekbdnsPVLE9zsO8/2Owzz26WZahf7PIp N/LHp035SR1laonJzcykrKyMkJKTc9pCQELZt21Ypx/jkk0/Iy8tj PDhjm3du3fHy8uLiRMn8swzz2CaJpMmTaKsrIxDhw6dcz/z58 nd /eNGrU6LzHmjZtGk888USl5BYRscpvt/3O3Pprza7DJ1i5NYdVW3P4ce9RtmUdd9z6C/R25bqWwdzQOphrWgThpU/7SR1S6/9pnz9/Pn369CEsLMyxLSgoiCVLljB69GhmzpyJzWbjzjvvpHPnzthsZ08TO3DgAMuWLeP999//w2NNnjyZhIQEx88FBQWEh4dX3smIiFxmhmHQPNiH5sE jOrZjGOFJXy7I4eVW3P4fvthck UsCTtAEvSDui2n9Q5lpaowMBAnJycyn0qDiA7O/u8k8Yvxt69e1m5ciUfffTRWc/16tWLXbt2kZubi7OzM/7 /oSGhhIZGXnW2MTEROrXr8 tt976h8dzc3Mrd4tQRKS2CfByZUCnRgzo1IiS0/ zyKdu 0kdZGmJcnV1pUuXLiQlJdG/f3/gzMTypKQkHn744Uvef2JiIsHBwfTt2/e8YwIDAwFYtWoVOTk5ZxUl0zRJTExk2LBhuLi4XHImEZHawtXZxlXNA7nqd7f9krZmk7b3WLnbfh3C/Vl4bxQ 7vrvqNQelt/OS0hIIC4ujq5duxIVFcWMGTMoLCx0TOAeNmwYDRs2ZNq0acCZieJbtmxx/DkzM5OMjAy8vb1p3ry5Y792u53ExETi4uJwdj77NBMTE2ndujVBQUEkJyczbtw4xo8fT8uWLcuNW7VqFbt37 a   6rql BiEiN90e3/b7dlsOG/Xk89E46bwzvhouT5avriFQKy5c4AJg9ezbTp08nKyuLjh07MnPmTKKjowG49tpriYiI4M033wRgz549NG3a9Kx99OzZk2 //dbx8/Lly nduzfbt2/niiuuOGv8pEmTePPNNzl69CgRERGMGjWK8ePHOz7B95u77rqLvXv3smbNmos Ly1xICICmzLzGfxKMkUlZQzq0ojnb29/1n9rRaoTfe1LNaASJSJyxjfbcrh3QSp2ExJuvIK/3dDC6kgi51WjvvZFRERqt taBfNk/3YAvLRiBx mHbA4kcilU4kSEZHLYmh0E0b1bAbAxA9/Zu0vuRYnErk0KlEiInLZPNK7Jf06hHHabvLgwjR2ZB 3OpJIhalEiYjIZWOzGUy/vT1REfU4fuo0w99IIbvglNWxRCpEJUpERC4rdxcnXh3WhcggLw7mn2Lkm6kUFp/ 8xeKVDMqUSIictn5e7ry5vAo6nu5svlgAQ8vSud0md3qWCIXRSVKREQs0bi J/OHd8PdxcY32w8z5dPNaNUdqUlUokRExDIdw/2ZOaQThgHvpuxj7ne7rI4kcsFUokRExFK92oYy9ZY2ADz/9XY zci0OJHIhVGJEhERyw2/qin3Xn3mK73 vuRn1v96xOJEIn9OJUpERKqFR29uTZ92oZSU2Xng7TR yTlhdSSRP6QSJSIi1YLNZvCvOzrSubE/ SdLGZ6YwuHjxVbHEjkvlSgREak23F2ceG1YV5rU9 TAsZPctyCVohKtISXVk0qUiIhUK/W93XhzRBQBni5sOJDP397NoMyupQ k lGJEhGRaqdpoBevx3XF1dnGyq3Z/N/nWkNKqh VKBERqZa6NKnHjDs6YhiwIHkv81fvtjqSSDkqUSIiUm3dfGUDHr25NQBPf7mVLzcesjiRyH pRImISLV279VNiYtpgmlC/OIM0vYetTqSCKASJSIi1ZxhGDzWry2xrUMoOW3nvgU/sju30OpYIipRIiJS/TnZDGbe2ZEOjfw4VlTKiMQUjpzQGlJiLZUoERGpETxdnXk9rhvh9TzYc6SI 976kVOlZVbHkjpMJUpERGqMIB83EodH4efhwk/78hi/OAO71pASi6hEiYhIjdI82JtX7 mCq5ONrzZl8cyXW62OJHWUSpSIiNQ40ZH1mT6oPQCvr97Nm2u0hpRcfipRIiJSI/21Y0MeuaklAE8s3cLyzVkWJ5K6RiVKRERqrNE9m3FnVGNME/723k9k7M zOpLUISpRIiJSYxmGwZN/bct1LYM4VWrn3jdT2XekyOpYUkeoRImISI3m7GRj9l2daRvmy5HCEoa/mUJeUYnVsaQOUIkSEZEaz8vNmTeGdyPMz51fDxdyv9aQkstAJUpERGqFEF933hwZhY 7M6l7jjFhyQatISVVSiVKRERqjStCfHjl7i64OBks/fkQzy/bbnUkqcVUokREpFbp0TyQZ287s4bUvO92sXDdXosTSW2lEiUiIrXOwC6NSLjxCgAe 3QTq7ZlW5xIaiPLS9ScOXOIiIjA3d2d6OhoUlJSzjt28 bNDBw4kIiICAzDYMaMGWeN e253z/GjBnjGLNr1y4GDBhAUFAQvr6 DB48mOzss/8F  KLL4iOjsbDw4OAgAD69 9fGacsIiKXwdjrmzO4ayPsJjy86Cc2Hsi3OpLUMpaWqMWLF5OQkMDUqVNJT0 nQ4cO9O7dm5ycnHOOLyoqIjIykmeffZbQ0NBzjklNTeXQoUOOx4oVKwAYNGgQAIWFhfTq1QvDMFi1ahVr1qyhpKSEfv36YbfbHfv58MMPueeeexgxYgQbNmxgzZo13HXXXZX8GxARkapiGAZPD7iSa1oEUlRSxsgFqRw4pjWkpPIYpmla9tGF6OhounXrxuzZswGw2 2Eh4czduxYJk2a9IevjYiIID4 nvj4 D8cFx8fz9KlS9m5cyeGYbB8 XL69OnDsWPH8PX1BSA/P5 AgACWL19ObGwsp0 fJiIigieeeIJ77733gs nuLiY4uJix88FBQWEh4eTn5/vOJaIiFxex0 VMmheMtuyjtMi2JsPRvfAz8PF6lhSjRUUFODn5/en79 WXYkqKSkhLS2N2NjY/4ax2YiNjSU5ObnSjrFw4UJGjhyJYRjAmaJjGAZubm6Oce7u7thsNlavXg1Aeno6mZmZ2Gw2OnXqRIMGDejTpw bNm36w NNmzYNPz8/xyM8PLxSzkNERCrOx92FxBHdCPV1Z2fOCUa9nUbJafufv1DkT1hWonJzcykrKyMkJKTc9pCQELKyKudLJD/55BPy8vIYPny4Y1v37t3x8vJi4sSJFBUVUVhYyIQJEygrK PQoUMA/PrrrwA8/vjj/POf/2Tp0qUEBARw7bXXcvTo0fMeb/LkyeTn5zse /fvr5TzEBGRS9PAz4M3hnfD282Z5F PMPHDn7HwRozUEpZPLK9K8 fPp0 fPoSFhTm2BQUFsWTJEj7//HO8vb3x8/MjLy Pzp07Y7Od XX8Njfq0UcfZeDAgXTp0oXExEQMw2DJkiXnPZ6bmxu vr7lHiIiUj20CfNlztDOONkMPv4pk5dW7LA6ktRwzlYdODAwECcnp7M FZednX3eSeMXY /evaxcuZKPPvrorOd69erFrl27yM3NxdnZGX9/f0JDQ4mMjASgQYMGALRp08bxGjc3NyIjI9m3b98lZxMREWv0vCKIZwa0Y KHG5m16hfCAzwZ3E1TL6RiLLsS5erqSpcuXUhKSnJss9vtJCUlERMTc8n7T0xMJDg4mL59 553TGBgIP7 /qxatYqcnBxuvfVWALp06YKbmxvbt/93pdvS0lL27NlDkyZNLjmbiIhY545ujRl7fXMAJn 8ke93HLY4kdRUll2JAkhISCAuLo6uXbsSFRXFjBkzKCwsZMSIEQAMGzaMhg0bMm3aNODMRPEtW7Y4/pyZmUlGRgbe3t40b97csV 73U5iYiJxcXE4O599iomJibRu3ZqgoCCSk5MZN24c48ePp2XLlgD4 voyatQopk6dSnh4OE2aNGH69OnAf5dKEBGRmivhxis4cOwkH/ UyUPvpPP gzG0CdMUDLk4lpaoO 64g8OHD/PYY4 RlZVFx44d frrrx2Tzfft2 eYpwRw8OBBOnXq5Pj5hRde4IUXXqBnz558  23ju0rV65k3759jBw58pzH3b59O5MnT bo0aNERETw6KOPMn78 HJjpk fjrOzM/fccw8nT54kOjqaVatWERAQUIm/ARERsYJhGDw3sD1Z adI/vUII99M5eMxPWjg52F1NKlBLF0nqra70HUmRETEGvknS7l97lp25pygVagPS0bF4OOuNaTqumq/TpSIiIjV/DzOrCEV5OPGtqzjPPROOqVlWkNKLoxKlIiI1GmNAjx5I64bnq5O/LAzl398tFFrSMkFUYkSEZE678pGfsy5qzM2A5akHWDWql sjiQ1gEqUiIgIcF2rYP7vr 0AeGnFDj5MO2BxIqnuVKJERET 4 7uTRjVsxkAEz/8mbW/5FqcSKozlSgREZH/8UjvlvTrEMZpu8mDC9PYkX3c6khSTalEiYiI/A bzWD67e3pFhHA8VOnGf5GCtkFp6yOJdWQSpSIiMjvuLs48dqwrkQGeXEw/xQj30ylsPi01bGkmlGJEhEROQd/T1feHB5FfS9XNh8s4OFF6ZzWGlLyP1SiREREzqNxfU/mD  Gu4uNb7YfZsqnm7DbtYaUnKESJSIi8gc6hvszc0gnDAPeTdnP2Pd 4lRpmdWxpBpQiRIREfkTvdqG8q/BHXFxMvji50Pc/fp6jhaWWB1LLKYSJSIicgH6d2rIgpFR Lg78 PeYwycu5Y9uYVWxxILqUSJiIhcoB7NAvlodA8a nuwO7eQ2 auJW3vMatjiUVUokRERC5CixAfPh7Tgysb nG0sIS7XlvHVxsPWR1LLKASJSIicpGCfdxZ/GB3YlsHU3zazkOL0nn9h18xTX1yry5RiRIREakAT1dnXrmnK8NimmCa8NQXW5n62WbKtARCnaESJSIiUkFONoMnbm3LP/u2xjDgreS9PPj2jxSVaHXzukAlSkRE5BIYhsF910Ty8l2dcXO2sXJrDkNeXUfOcX3fXm2nEiUiIlIJ lzZgEX3d6eelys/H8hnwJy17Mw bnUsqUIqUSIiIpWkS5MAPhrdg6aBXmTmneS2uWtZuyvX6lhSRVSiREREKlFEoBcfju5B1yYBHD91mrg3Uvj4pwNWx5IqoBIlIiJSyep5ubLwvmj6tm9AaZnJ MUbmJm0U0sg1DIqUSIiIlXA3cWJWUM68WDPSABeWrGDRz74mdIyu8XJpLKoRImIiFQRm81gcp/WPNW/HTYDlqQdYERiKgWnSq2OJpVAJUpERKSK3d29CfPjuuHp6sTqX3IZPC Zg3knrY4ll0glSkRE5DK4rlUw7z8YQ7CPG9uyjjPg5TVsPphvdSy5BCpRIiIil0m7hn58POYqrgjxJrugmMHzkvlme47VsaSCVKJEREQuo4b HnwwugdXNa9PYUkZ9y34kUXr91kdSypAJUpEROQy83V3IXF4FLd3aUSZ3eQfH2/kua 3YdeXF9coKlEiIiIWcHW2Mf329oyPvQKAud/uYtziDE6VllmcTC6USpSIiIhFDMNgXGwLXhzUAWebwecbDnLP/PUcKyyxOppcgGpRoubMmUNERATu7u5ER0eTkpJy3rGbN29m4MCBREREYBgGM2bMOGvMb8/9/jFmzBjHmF27djFgwACCgoLw9fVl8ODBZGdn/ l nn322Uo7bxEREYCBXRrx1sgofNydSd1zjIFz17LvSJHVseRPWF6iFi9eTEJCAlOnTiU9PZ0OHTrQu3dvcnLO/WmFoqIiIiMjefbZZwkNDT3nmNTUVA4dOuR4rFixAoBBgwYBUFhYSK9evTAMg1WrVrFmzRpKSkro168fdnv5lWT/7// r9y xo4dW4lnLyIickaP5oF8OLoHDf09 DW3kAEvryF93zGrY8kfMEyLv8gnOjqabt26MXv2bADsdjvh4eGMHTuWSZMm/eFrIyIiiI PJz4 /g/HxcfHs3TpUnbu3IlhGCxfvpw ffpw7NgxfH19AcjPzycgIIDly5cTGxt7Ufv/TXFxMcXFxY6fCwoKCA8PJz8/33EcERGRP5JTcIqRC1LZlFmAm7ONfw/pxE3tzn3RQKpGQUEBfn5 f/r bemVqJKSEtLS0hylBcBmsxEbG0tycnKlHWPhwoWMHDkSwzCAM2XHMAzc3Nwc49zd3bHZbKxevbrc65999lnq169Pp06dmD59OqdPnz7vsaZNm4afn5/jER4eXinnICIidUewrzuLH4jh lbBFJ 2M/qdNOav3m11LDkHS0tUbm4uZWVlhISElNseEhJCVlZWpRzjk08 IS8vj HDhzu2de/eHS8vLyZOnEhRURGFhYVMmDCBsrIyDh065Bj3t7/9jffee49vvvmGBx98kGeeeYZHHnnkvMeaPHky fn5jsf /fsr5RxERKRu8XJz5tV7unB398aYJjy5dAuPf7aZMi2BUK1YPieqqs2fP58 ffoQFhbm2BYUFMSSJUv4/PPP8fb2xs/Pj7y8PDp37ozN9t9fSUJCAtdeey3t27dn1KhRvPjii8yaNavcLbv/5ebmhq vb7mHiIhIRTg72Xjyr 34x82tAHhz7R5GLUyjqOT8d0Tk8rK0RAUGBuLk5HTWp Kys7PPO2n8Yuzdu5eVK1dy3333nfVcr1692LVrFzk5OeTm5vL222 TmZlJZGTkefcXHR3N6dOn2bNnzyVnExER TOGYfDAX5ox567OuDrbWLElmztfXcfh4 f n3m5vCwtUa6urnTp0oWkpCTHNrvdTlJSEjExMZe8/8TERIKDg nbt 95xwQGBuLv78 qVavIycnh1ltvPe/YjIwMbDYbwcHBl5xNRETkQvVt34B3748mwNOFDQfyGfDyGn7JOW51rDrP2eoACQkJxMXF0bVrV6KiopgxYwaFhYWMGDECgGHDhtGwYUOmTZsGnJkovmXLFsefMzMzycjIwNvbm bNmzv2a7fbSUxMJC4uDmfns08zMTGR1q1bExQURHJyMuPGjWP8 PG0bNkSgOTkZNavX891112Hj48PycnJjB8/nrvvvpuAgICq/rWIiIiU06VJPT566CpGJKaw50gRt728lleHdaV7ZH2ro9VdZjUwa9Yss3Hjxqarq6sZFRVlrlu3zvFcz549zbi4OMfPu3fvNoGzHj179iy3z2XLlpmAuX379nMec LEiWZISIjp4uJitmjRwnzxxRdNu93ueD4tLc2Mjo42/fz8THd3d7N169bmM888Y546deqCzys/P98EzPz8/At jYiIyB85cqLYHDBntdlk4lKzxT  ND/56YDVkWqdC33/tnydqNrsQteZEBERuRinSstIeD DLzee ST7hF5XMOa65o6lfOTS1Ih1okREROTiubs4MfvOzjz4lzMfhnph Q4mf7SR0jL7n7xSKpNKlIiISA1ksxlMvrk1T/61LTYD3kvdz8g3Uzl qtTqaHWGSpSIiEgNdk9MBK8N64qHixM/7Mxl0LxkDuWftDpWnaASJSIiUsPd0DqE9x MIcjHjW1ZxxkwZy1bDhZYHavWU4kSERGpBa5s5MfHD/WgRbA3WQWnGPxKMt/tOGx1rFpNJUpERKSWaBTgyQejexATWZ8TxacZ WYq76XsszpWraUSJSIiUov4ebiwYGQUt3VuSJndZNJHG5m bBta0ajyqUSJiIjUMq7ONl4c1IFxN7QAYM43u4hfnEHx6TKLk9UuKlEiIiK1kGEYjL/xCqbf3h5nm8GnGQe5Z34KeUUlVkerNVSiREREarFBXcN5c0QUPm7OpOw ysC5a9l/tMjqWLWCSpSIiEgtd3WLQD4Y3YMwP3d2HS6k/5w1fLM9x pYNZ5KlIiISB3QMtSHj8dcRbuGvhwpLGFEYipPfL6ZU6WaJ1VRKlEiIiJ1RIivOx M6sHwHhEAJK7ZQ/85a9iRfdzaYDWUSpSIiEgd4u7ixOO3tiVxeDfqe7myLes4/Wat5u3kPVoG4SKpRImIiNRB17UK5qv4a h5RRDFp 1M XQz97/1I0dOFFsdrcZQiRIREamjgn3cSRzejcduaYOrk42VW3O46d8/8MNOfV3MhVCJEhERqcNsNoORVzflkzFX0SLYm8PHi7lnfgpPf7FFi3P CZUoERERoU2YL589fDX3dG8CwGs/7GbAnLX8knPC4mTVl0qUiIiIAODh6sST/dvx2rCuBHi6sOVQAbfM oFF6/dp0vk5qESJiIhIOTe2CeHr L9wdfNATpXa cfHGxm1MI1jhfrKmP lEiUiIiJnCfF1562RUTx6c2tcnAyWbc7mpn9/z9pfcq2OVm2oRImIiMg52WwG9/8lko8fuorIIC yC4oZOn89z361jZLTdqvjWU4lSkRERP5Qu4Z LB17NXdGNcY0Yd53uxg4dy2/Hq7bk85VokRERORPebo6M 22K5l3dxf8PV3YmJnPLbNW837q/jo76VwlSkRERC7YTe1C WrcNcRE1qeopIxHPvyZMYvSyS8qtTraZacSJSIiIhelgZ8HC  LZuJNrXC2GXy5MYs //6e9b8esTraZaUSJSIiIhfNyWYw tpmfDi6BxH1PTmYf4ohr63jhWXbKS2rG5POVaJERESkwjqE /PF365hcNdGmCbM/uYXBs1LZu RQqujVTmVKBEREbkkXm7OPH97B2bf1Qkfd2cy9udx879/4MO0A7V60rlKlIiIiFSKW9qH8XX8X4iKqEdhSRn/b8kGxr2XQcGp2jnpXCVKREREKk1Dfw/efaA7E3pdgZPN4LMNB kz4wd 3HPU6miVTiVKREREKpWTzeDh61uwZFQMjet5kpl3ksGvJPOvFTs4XYsmnVeLEjVnzhwiIiJwd3cnOjqalJSU847dvHkzAwcOJCIiAsMwmDFjxlljfnvu948xY8Y4xuzatYsBAwYQFBSEr68vgwcPJjs7 5zHLC4upmPHjhiGQUZGxqWeroiISJ3QuXEAX/ztam7r1BC7Cf9O2skdr65j/9Eiq6NVCstL1OLFi0lISGDq1Kmkp6fToUMHevfuTU5OzjnHFxUVERkZybPPPktoaOg5x6SmpnLo0CHHY8WKFQAMGjQIgMLCQnr16oVhGKxatYo1a9ZQUlJCv379sNvPbsiPPPIIYWFhlXTGIiIidYePuwsv3dGRfw/piI bM2l7j3Hzv3/g04xMq6NdMsO0eNp8dHQ03bp1Y/bs2QDY7XbCw8MZO3YskyZN sPXRkREEB8fT3x8/B Oi4 PZ nSpezcuRPDMFi fDl9 vTh2LFj Pr6ApCfn09AQADLly8nNjbW8dqvvvqKhIQEPvzwQ9q2bctPP/1Ex44dL jcCgoK8PPzIz8/33EcERGRumr/0SLiF2eQtvcYALd1asgTf22Lj7uLxcnKu9D3b0uvRJWUlJCWllautNhsNmJjY0lOTq60YyxcuJCRI0diGAZw5vacYRi4ubk5xrm7u2Oz2Vi9erVjW3Z2Nvfffz9vv/02np6ef3qs4uJiCgoKyj1ERETkjPB6nix oDvxsS2wGfDRT5n0nbma9H3HrI5WIZaWqNzcXMrKyggJCSm3PSQkhKysrEo5xieffEJeXh7Dhw93bOvevTteXl5MnDiRoqIiCgsLmTBhAmVlZRw6dAgA0zQZPnw4o0aNomvXrhd0rGnTpuHn5 d4hIeHV8o5iIiI1BbOTjbiY6/g/QdjaOjvwb6jRQyal8yspJ2U2WvWmlKWz4mqavPnz6dPnz7l5jQFBQWxZMkSPv/8c7y9vfHz8yMvL4/OnTtjs535lcyaNYvjx48zefLkCz7W5MmTyc/Pdzz2799f6ecjIiJSG3SNqMdX8ddwa4cwyuwmL67YwZ2vriMz76TV0S6YpSUqMDAQJyensz4Vl52dfd5J4xdj7969rFy5kvvuu  s53r16sWuXbvIyckhNzeXt99 m8zMTCIjIwFYtWoVycnJuLm54ezsTPPmzQHo2rUrcXFx5zyem5sbvr6 5R4iIiJybr7uLvx7SEdeGtwBL1cnUvYcpc M71n680Gro10QS0uUq6srXbp0ISkpybHNbreTlJRETEzMJe8/MTGR4OBg vbte94xgYGB Pv7s2rVKnJycrj11lsBmDlzJhs2bCAjI4OMjAy /PJL4MynCZ9  ulLziYiIiJgGAa3dW7El OuoWO4PwWnTvPwop/4 5INFBaftjreH3K2OkBCQgJxcXF07dqVqKgoZsyYQWFhISNGjABg2LBhNGzYkGnTpgFnJopv2bLF8efMzEwyMjLw9vZ2XC2CM2UsMTGRuLg4nJ3PPs3ExERat25NUFAQycnJjBs3jvHjx9OyZUsAGjduXG68t7c3AM2aNaNRo0aV/4sQERGpw5rU92LJqBj vXInc779hSVpB0jdc5R/D lEh3B/q Odk Ul6o477uDw4cM89thjZGVl0bFjR77  mvHZPN9 /Y55ikBHDx4kE6dOjl fuGFF3jhhRfo2bMn3377rWP7ypUr2bdvHyNHjjzncbdv387kyZM5evQoERERPProo4wfP75qTlJERET lIuTjQm9W3JNi0DGL85gz5EiBs5dS0KvK3jwL81wshlWRyzH8nWiajOtEyUiIlIx UWl/OPjjXyx8cyn5rtH1uNfd3SkgZ9HlR 7RqwTJSIiInIufp4uzL6rE8/f3h5PVyfW/XqUm2b8wNebDlkdzUElSkRERKolwzAY3DWcL/52De0b ZF/spRRC9OZ/NHPFJVYP lcJUpERESqtaaBXnwwqgejr22GYcC7Kfu5ZdZqNmXmW5pLJUpERESqPVdnGxNvasU790UT6uvOr4cLGfDyGku/yFglSkRERGqMHs0C WrcNfRuG4KLk40rG/pZlsXyJQ5ERERELkaAlyvz7u7C7txCIoO8LcuhK1EiIiJS4xiGYWmBApUoERERkQpRiRIRERGpAJUoERERkQpQiRIRERGpAJUoERERkQpQiRIRERGpAJUoERERkQpQiRIRERGpAJUoERERkQpQiRIRERGpAJUoERERkQpQiRIRERGpAJUoERERkQpwtjpAbWaaJgAFBQUWJxEREZEL9dv79m/v4 ejElWFjh8/DkB4eLjFSURERORiHT9 HD8/v/M b5h/VrOkwux2OwcPHsTHxwfDMCptvwUFBYSHh7N//358fX0rbb9Scfo7qV7091G96O jetHfx58zTZPjx48TFhaGzXb mU 6ElWFbDYbjRo1qrL9 /r66l AakZ/J9WL/j6qF/19VC/6 /hjf3QF6jeaWC4iIiJSASpRIiIiIhWgElUDubm5MXXqVNzc3KyOIv hv5PqRX8f1Yv PqoX/X1UHk0sFxEREakAXYkSERERqQCVKBEREZEKUIkSERERqQCVKBEREZEKUImqgebMmUNERATu7u5ER0eTkpJidaQ6adq0aXTr1g0fHx Cg4Pp378/27dvtzqW/Mezzz6LYRjEx8dbHaXOyszM5O6776Z /fp4eHhw5ZVX8uOPP1odq84qKytjypQpNG3aFA8PD5o1a8aTTz75p98PJ enElXDLF68mISEBKZOnUp6ejodOnSgd /e5OTkWB2tzvnuu 8YM2YM69atY8WKFZSWltKrVy8KCwutjlbnpaam8sorr9C fXuro9RZx44d46qrrsLFxYWvvvqKLVu28OKLLxIQEGB1tDrrueeeY 7cucyePZutW7fy3HPP8fzzzzNr1iyro9VYWuKghomOjqZbt27Mnj0bOPP9fOHh4YwdO5ZJkyZZnK5uO3z4MMHBwXz33Xf85S9/sTpOnXXixAk6d 7Myy /zFNPPUXHjh2ZMWOG1bHqnEmTJrFmzRp  OEHq6PIf9xyyy2EhIQwf/58x7aBAwfi4eHBwoULLUxWc lKVA1SUlJCWloasbGxjm02m43Y2FiSk5MtTCYA fn5ANSrV8/iJHXbmDFj6Nu3b7l/T Ty  yzz jatSuDBg0iODiYTp068dprr1kdq07r0aMHSUlJ7NixA4ANGzawevVq vTpY3GymktfQFyD5ObmUlZWRkhISLntISEhbNu2zaJUAmeuCMbHx3PVVVfRrl07q PUWe 99x7p6emkpqZaHaXO /XXX5k7dy4JCQn84x//IDU1lb/97W 4uroSFxdndbw6adKkSRQUFNCqVSucnJwoKyvj6aefZujQoVZHq7FUokQqwZgxY9i0aROrV6 2OkqdtX//fsaNG8eKFStwd3e3Ok6dZ7fb6dq1K8888wwAnTp1YtOmTcybN08lyiLvv/8 77zzDosWLaJt27ZkZGQQHx9PWFiY/k4qSCWqBgkMDMTJyYns7Oxy27OzswkNDbUolTz88MMsXbqU77//nkaNGlkdp85KS0sjJyeHzp07O7aVlZXx/fffM3v2bIqLi3FycrIwYd3SoEED2rRpU25b69at fDDDy1KJH// 9 ZNGkSQ4YMAeDKK69k7969TJs2TSWqgjQnqgZxdXWlS5cuJCUlObbZ7XaSkpKIiYmxMFndZJomDz/8MB9//DGrVq2iadOmVkeq02644QY2btxIRkaG49G1a1eGDh1KRkaGCtRldtVVV5215MeOHTto0qSJRYmkqKgIm638276TkxN2u92iRDWfrkTVMAkJCcTFxdG1a1eioqKYMWMGhYWFjBgxwupodc6YMWNYtGgRn376KT4 PmRlZQHg5 eHh4eHxenqHh8fn7Pmo3l5eVG/fn3NU7PA PHj6dGjB8888wyDBw8mJSWFV199lVdffdXqaHVWv379ePrpp2ncuDFt27blp59 4qWXXmLkyJFWR6uxtMRBDTR79mymT59OVlYWHTt2ZObMmURHR1sdq84xDOOc2xMTExk fPjlDSPndO2112qJAwstXbqUyZMns3PnTpo2bUpCQgL333 /1bHqrOPHjzNlyhQ /vhjcnJyCAsL48477 Sxxx7D1dXV6ng1kkqUiIiISAVoTpSIiIhIBahEiYiIiFSASpSIiIhIBahEiYiIiFSASpSIiIhIBahEiYiIiFSASpSIiIhIBahEiYiIiFSASpSIyGVkGAaffPKJ1TFEpBKoRIlInTF8 HAMwzjrcdNNN1kdTURqIH0BsYjUKTfddBOJiYnltrm5uVmURkRqMl2JEpE6xc3NjdDQ0HKPgIAA4Myttrlz59KnTx88PDyIjIzkgw8 KPf6jRs3cv311 Ph4UH9 vV54IEHOHHiRLkxb7zxBm3btsXNzY0GDRrw8MMPl3s NzeXAQMG4OnpSYsWLfjss8 q9qRFpEqoRImI/I8pU6YwcOBANmzYwNChQxkyZAhbt24FoLCwkN69exMQEEBqaipLlixh5cqV5UrS3LlzGTNmDA888AAbN27ks88 o3nz5uWO8cQTTzB48GB /vlnbr75ZoYOHcrRo0cv63mKSCUwRUTqiLi4ONPJycn08vIq93j66adN0zRNwBw1alS510RHR5ujR482TdM0X331VTMgIMA8ceKE4/kvvvjCtNlsZlZWlmmaphkWFmY  uij580AmP/85z8dP584ccIEzK   qrSzlNELg/NiRKROuW6665j7ty55bbVq1fP8eeYmJhyz8XExJCRkQHA1q1b6dChA15eXo7nr7rqKux2O9u3b8cwDA4ePMgNN9zwhxnat2/v LOXlxe vr7k5ORU9JRExCIqUSJSp3h5eZ11e62yeHh4XNA4FxeXcj8bhoHdbq KSCJShTQnSkTkf6xbt 6sn1u3bg1A69at2bBhA4WFhY7n16xZg81mo2XLlvj4 BAREUFSUtJlzSwi1tCVKBGpU4qLi8nKyiq3zdnZmcDAQACWLFlC165dufrqq3nnnXdISUlh/vz5AAwdOpSpU6cSFxfH448/zuHDhxk7diz33HMPISEhADz  OOMGjWK4OBg vTpw/Hjx1mzZg1jx469vCcqIlVOJUpE6pSvv/6aBg0alNvWsmVLtm3bBpz55Nx7773HQw89RIMGDXj33Xdp06YNAJ6enixbtoxx48bRrVs3PD09GThwIC 99JJjX3FxcZw6dYp//etfTJgwgcDAQG6//fbLd4IictkYpmmaVocQEakODMPg448/pn///lZHEZEaQHOiRERERCpAJUpERESkAjQnSkTkPzS7QUQuhq5EiYiIiFSASpSIiIhIBahEiYiIiFSASpSIiIhIBahEiYiIiFSASpSIiIhIBahEiYiIiFSASpSIiIhIBfx/D0ToORHh9pAAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ]
    }
  ]
}