CIFAR_10C_Evaluation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "environment": {
      "name": "tf2-gpu.2-4.mnightly-2021-01-20-debian-10-test",
      "type": "gcloud",
      "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-4:mnightly-2021-01-20-debian-10-test"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.9"
    },
    "colab": {
      "name": "CIFAR_10C_Evaluation.ipynb",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sayakpaul/Consistency-Training-with-Supervision/blob/main/CIFAR_10C_Evaluation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "filled-jurisdiction"
      },
      "source": [
        "## Setup"
      ],
      "id": "filled-jurisdiction"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "liberal-edmonton"
      },
      "source": [
        "# All model weights\n",
        "!wget https://git.io/JOKI9 -O consistency_training_model_weights.zip"
      ],
      "id": "liberal-edmonton",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "three-niger"
      },
      "source": [
        "from tensorflow.keras import layers\n",
        "import tensorflow as tf\n",
        "\n",
        "import tensorflow_datasets as tfds\n",
        "tfds.disable_progress_bar()\n",
        "\n",
        "from tqdm import tqdm\n",
        "import numpy as np"
      ],
      "id": "three-niger",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "residential-gossip"
      },
      "source": [
        "## Define Hyperparameters"
      ],
      "id": "residential-gossip"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "loose-devil"
      },
      "source": [
        "AUTO = tf.data.AUTOTUNE\n",
        "DATASET_NAME = \"cifar10_corrupted\"\n",
        "BATCH_SIZE = 128\n",
        "IMAGE_SIZE = 72"
      ],
      "id": "loose-devil",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "related-yahoo",
        "outputId": "b1cc4e9a-d29f-41c1-9558-b4477ff469ab"
      },
      "source": [
        "VERSIONS = [\n",
        "    \"brightness_5\",\n",
        "    \"contrast_5\",\n",
        "    \"defocus_blur_5\",\n",
        "    \"elastic_5\",\n",
        "    \"fog_5\",\n",
        "    \"frost_5\",\n",
        "    \"frosted_glass_blur_5\",\n",
        "    \"gaussian_blur_5\",\n",
        "    \"gaussian_noise_5\",\n",
        "    \"impulse_noise_5\",\n",
        "    \"jpeg_compression_5\",\n",
        "    \"motion_blur_5\",\n",
        "    \"pixelate_5\",\n",
        "    \"saturate_5\",\n",
        "    \"shot_noise_5\",\n",
        "    \"snow_5\",\n",
        "    \"spatter_5\",\n",
        "    \"speckle_noise_5\",\n",
        "    \"zoom_blur_5\"\n",
        "]\n",
        "\n",
        "print(f\"Total sub-versions of the CIFAR10-C dataset: {len(VERSIONS)}\")"
      ],
      "id": "related-yahoo",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total sub-versions of the CIFAR10-C dataset: 19\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "responsible-techno"
      },
      "source": [
        "## Utilities"
      ],
      "id": "responsible-techno"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dedicated-typing"
      },
      "source": [
        "def prepare_dataset(ds):\n",
        "    ds = (ds\n",
        "          .batch(BATCH_SIZE)\n",
        "          .map(lambda x, y: (tf.image.resize(x, (IMAGE_SIZE, IMAGE_SIZE)), y), \n",
        "               num_parallel_calls=AUTO)\n",
        "          .prefetch(AUTO)\n",
        "    )\n",
        "    return ds"
      ],
      "id": "dedicated-typing",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "designing-chancellor"
      },
      "source": [
        "def get_training_model(num_classes=10):\n",
        "    resnet50_v2 = tf.keras.applications.ResNet50V2(\n",
        "        weights=None,\n",
        "        include_top=False,\n",
        "        input_shape=(IMAGE_SIZE, IMAGE_SIZE, 3),\n",
        "    )\n",
        "    model = tf.keras.Sequential(\n",
        "        [\n",
        "            layers.Input((IMAGE_SIZE, IMAGE_SIZE, 3)),\n",
        "            layers.experimental.preprocessing.Rescaling(scale=1.0 / 127.5, offset=-1),\n",
        "            resnet50_v2,\n",
        "            layers.GlobalAveragePooling2D(),\n",
        "            layers.Dense(num_classes)\n",
        "        ]\n",
        "    )\n",
        "    return model"
      ],
      "id": "designing-chancellor",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dangerous-processing"
      },
      "source": [
        "def evaluate_model(model):\n",
        "    acc_dict = {}\n",
        "    for version in tqdm(VERSIONS):\n",
        "        print(f\"Processing {version}\")\n",
        "        dataset_fullname = DATASET_NAME + \"/\" + version\n",
        "        loaded_ds = tfds.load(\n",
        "            dataset_fullname,\n",
        "            split=\"test\",\n",
        "            as_supervised=True\n",
        "        )\n",
        "        loaded_ds = prepare_dataset(loaded_ds)\n",
        "        _, acc = model.evaluate(loaded_ds, verbose=0)\n",
        "        print(f\"Test accuracy on {version}: {acc*100}%\")\n",
        "        acc_dict[version] = acc*100\n",
        "        \n",
        "    return acc_dict, np.mean(list(acc_dict.values()))"
      ],
      "id": "dangerous-processing",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "monthly-mount"
      },
      "source": [
        "## Evaluation"
      ],
      "id": "monthly-mount"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rural-attitude"
      },
      "source": [
        "### SWA"
      ],
      "id": "rural-attitude"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "essential-height",
        "outputId": "6e029f58-9f51-490b-d8a4-f53a97186ba4"
      },
      "source": [
        "# Evaluate teacher model trained with SWA\n",
        "teacher_model_swa = get_training_model()\n",
        "teacher_model_swa.load_weights(\"teacher_model_swa.h5\")\n",
        "teacher_model_swa.compile(loss=\"sparse_categorical_crossentropy\",\n",
        "                         metrics=[\"accuracy\"])\n",
        "acc_dict, mean_top_1 = evaluate_model(teacher_model_swa)\n",
        "print(f\"Mean Top-1 Accuracy: {mean_top_1}%\")"
      ],
      "id": "essential-height",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 0/19 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing brightness_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  5%|▌         | 1/19 [00:17<05:20, 17.80s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on brightness_5: 76.05999708175659%\n",
            "Processing contrast_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 11%|█         | 2/19 [00:19<02:22,  8.40s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on contrast_5: 24.34999942779541%\n",
            "Processing defocus_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 16%|█▌        | 3/19 [00:21<01:26,  5.42s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on defocus_blur_5: 71.59000039100647%\n",
            "Processing elastic_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 21%|██        | 4/19 [00:23<01:00,  4.02s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on elastic_5: 74.19999837875366%\n",
            "Processing fog_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 26%|██▋       | 5/19 [00:25<00:45,  3.25s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on fog_5: 48.28999936580658%\n",
            "Processing frost_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 32%|███▏      | 6/19 [00:27<00:36,  2.79s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frost_5: 62.26999759674072%\n",
            "Processing frosted_glass_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 37%|███▋      | 7/19 [00:28<00:29,  2.48s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frosted_glass_blur_5: 58.980000019073486%\n",
            "Processing gaussian_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 42%|████▏     | 8/19 [00:30<00:25,  2.28s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_blur_5: 66.60000085830688%\n",
            "Processing gaussian_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 9/19 [00:32<00:21,  2.15s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_noise_5: 41.909998655319214%\n",
            "Processing impulse_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 53%|█████▎    | 10/19 [00:34<00:18,  2.06s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on impulse_noise_5: 23.970000445842743%\n",
            "Processing jpeg_compression_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 58%|█████▊    | 11/19 [00:36<00:15,  2.00s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on jpeg_compression_5: 79.1599988937378%\n",
            "Processing motion_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 63%|██████▎   | 12/19 [00:38<00:13,  1.97s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on motion_blur_5: 65.77000021934509%\n",
            "Processing pixelate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 68%|██████▊   | 13/19 [00:40<00:11,  1.93s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on pixelate_5: 73.51999878883362%\n",
            "Processing saturate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 74%|███████▎  | 14/19 [00:42<00:09,  1.91s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on saturate_5: 69.6399986743927%\n",
            "Processing shot_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 79%|███████▉  | 15/19 [00:43<00:07,  1.89s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on shot_noise_5: 45.21999955177307%\n",
            "Processing snow_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 84%|████████▍ | 16/19 [00:45<00:05,  1.89s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on snow_5: 65.39999842643738%\n",
            "Processing spatter_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 89%|████████▉ | 17/19 [00:47<00:03,  1.87s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on spatter_5: 64.16000127792358%\n",
            "Processing speckle_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 95%|█████████▍| 18/19 [00:49<00:01,  1.86s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on speckle_noise_5: 45.62999904155731%\n",
            "Processing zoom_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 19/19 [00:51<00:00,  2.70s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on zoom_blur_5: 77.49000191688538%\n",
            "Mean Top-1 Accuracy: 59.695262579541456%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "instant-insurance",
        "outputId": "e62bb54f-d2e3-4e32-abc1-32bdd5f1a22f"
      },
      "source": [
        "# Evaluate the corresponding student model\n",
        "student_noisy_swa = get_training_model()\n",
        "student_noisy_swa.load_weights(\"student_noisy_swa.h5\")\n",
        "student_noisy_swa.compile(loss=\"sparse_categorical_crossentropy\",\n",
        "                         metrics=[\"accuracy\"])\n",
        "acc_dict, mean_top_1 = evaluate_model(student_noisy_swa)\n",
        "print(f\"Mean Top-1 Accuracy: {mean_top_1}%\")"
      ],
      "id": "instant-insurance",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 0/19 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing brightness_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  5%|▌         | 1/19 [00:02<00:51,  2.85s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on brightness_5: 82.91000127792358%\n",
            "Processing contrast_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 11%|█         | 2/19 [00:04<00:37,  2.22s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on contrast_5: 29.730001091957092%\n",
            "Processing defocus_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 16%|█▌        | 3/19 [00:06<00:32,  2.01s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on defocus_blur_5: 73.00000190734863%\n",
            "Processing elastic_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 21%|██        | 4/19 [00:08<00:28,  1.92s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on elastic_5: 74.29999709129333%\n",
            "Processing fog_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 26%|██▋       | 5/19 [00:09<00:26,  1.86s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on fog_5: 50.26000142097473%\n",
            "Processing frost_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 32%|███▏      | 6/19 [00:11<00:23,  1.83s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frost_5: 58.92000198364258%\n",
            "Processing frosted_glass_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 37%|███▋      | 7/19 [00:13<00:21,  1.81s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frosted_glass_blur_5: 59.78999733924866%\n",
            "Processing gaussian_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 42%|████▏     | 8/19 [00:15<00:19,  1.80s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_blur_5: 67.93000102043152%\n",
            "Processing gaussian_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 9/19 [00:17<00:17,  1.79s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_noise_5: 46.25999927520752%\n",
            "Processing impulse_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 53%|█████▎    | 10/19 [00:18<00:16,  1.78s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on impulse_noise_5: 30.98999857902527%\n",
            "Processing jpeg_compression_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 58%|█████▊    | 11/19 [00:20<00:14,  1.78s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on jpeg_compression_5: 76.39999985694885%\n",
            "Processing motion_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 63%|██████▎   | 12/19 [00:22<00:12,  1.77s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on motion_blur_5: 66.72000288963318%\n",
            "Processing pixelate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 68%|██████▊   | 13/19 [00:24<00:10,  1.77s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on pixelate_5: 73.94999861717224%\n",
            "Processing saturate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 74%|███████▎  | 14/19 [00:25<00:08,  1.77s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on saturate_5: 81.05000257492065%\n",
            "Processing shot_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 79%|███████▉  | 15/19 [00:27<00:07,  1.77s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on shot_noise_5: 51.88000202178955%\n",
            "Processing snow_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 84%|████████▍ | 16/19 [00:29<00:05,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on snow_5: 65.82000255584717%\n",
            "Processing spatter_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 89%|████████▉ | 17/19 [00:31<00:03,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on spatter_5: 70.31999826431274%\n",
            "Processing speckle_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 95%|█████████▍| 18/19 [00:32<00:01,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on speckle_noise_5: 53.46999764442444%\n",
            "Processing zoom_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 19/19 [00:34<00:00,  1.82s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on zoom_blur_5: 79.67000007629395%\n",
            "Mean Top-1 Accuracy: 62.80894765728399%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "stainless-prescription"
      },
      "source": [
        "### MA"
      ],
      "id": "stainless-prescription"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "limiting-scanning",
        "outputId": "5935e901-1288-4abb-bf87-77a295456c62"
      },
      "source": [
        "# Evaluate teacher model trained with MA\n",
        "teacher_model_ma = get_training_model()\n",
        "teacher_model_ma.load_weights(\"teacher_model_ma.h5\")\n",
        "teacher_model_ma.compile(loss=\"sparse_categorical_crossentropy\",\n",
        "                         metrics=[\"accuracy\"])\n",
        "acc_dict, mean_top_1 = evaluate_model(teacher_model_ma)\n",
        "print(f\"Mean Top-1 Accuracy: {mean_top_1}%\")"
      ],
      "id": "limiting-scanning",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 0/19 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing brightness_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  5%|▌         | 1/19 [00:02<00:50,  2.83s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on brightness_5: 73.14000129699707%\n",
            "Processing contrast_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 11%|█         | 2/19 [00:04<00:37,  2.21s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on contrast_5: 19.679999351501465%\n",
            "Processing defocus_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 16%|█▌        | 3/19 [00:06<00:32,  2.01s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on defocus_blur_5: 71.5499997138977%\n",
            "Processing elastic_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 21%|██        | 4/19 [00:08<00:28,  1.91s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on elastic_5: 74.76999759674072%\n",
            "Processing fog_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 26%|██▋       | 5/19 [00:09<00:25,  1.85s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on fog_5: 47.96999990940094%\n",
            "Processing frost_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 32%|███▏      | 6/19 [00:11<00:23,  1.81s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frost_5: 61.29999756813049%\n",
            "Processing frosted_glass_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 37%|███▋      | 7/19 [00:13<00:21,  1.79s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frosted_glass_blur_5: 61.41999959945679%\n",
            "Processing gaussian_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 42%|████▏     | 8/19 [00:15<00:19,  1.78s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_blur_5: 66.03000164031982%\n",
            "Processing gaussian_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 9/19 [00:16<00:17,  1.77s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_noise_5: 45.899999141693115%\n",
            "Processing impulse_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 53%|█████▎    | 10/19 [00:18<00:15,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on impulse_noise_5: 30.320000648498535%\n",
            "Processing jpeg_compression_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 58%|█████▊    | 11/19 [00:20<00:14,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on jpeg_compression_5: 78.4600019454956%\n",
            "Processing motion_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 63%|██████▎   | 12/19 [00:22<00:12,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on motion_blur_5: 64.19000029563904%\n",
            "Processing pixelate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 68%|██████▊   | 13/19 [00:23<00:10,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on pixelate_5: 72.26999998092651%\n",
            "Processing saturate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 74%|███████▎  | 14/19 [00:25<00:08,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on saturate_5: 66.04999899864197%\n",
            "Processing shot_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 79%|███████▉  | 15/19 [00:27<00:07,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on shot_noise_5: 49.36999976634979%\n",
            "Processing snow_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 84%|████████▍ | 16/19 [00:29<00:05,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on snow_5: 63.60999941825867%\n",
            "Processing spatter_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 89%|████████▉ | 17/19 [00:30<00:03,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on spatter_5: 63.8700008392334%\n",
            "Processing speckle_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 95%|█████████▍| 18/19 [00:32<00:01,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on speckle_noise_5: 49.75000023841858%\n",
            "Processing zoom_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 19/19 [00:34<00:00,  1.81s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on zoom_blur_5: 78.32000255584717%\n",
            "Mean Top-1 Accuracy: 59.89315792133934%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "legendary-berry",
        "outputId": "ea17c1ac-412a-4843-ccc9-4f742837db0b"
      },
      "source": [
        "# Evaluate the corresponding student model\n",
        "student_noisy_ma = get_training_model()\n",
        "student_noisy_ma.load_weights(\"student_noisy_ma.h5\")\n",
        "student_noisy_ma.compile(loss=\"sparse_categorical_crossentropy\",\n",
        "                         metrics=[\"accuracy\"])\n",
        "acc_dict, mean_top_1 = evaluate_model(student_noisy_ma)\n",
        "print(f\"Mean Top-1 Accuracy: {mean_top_1}%\")"
      ],
      "id": "legendary-berry",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 0/19 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing brightness_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  5%|▌         | 1/19 [00:02<00:51,  2.84s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on brightness_5: 81.80999755859375%\n",
            "Processing contrast_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 11%|█         | 2/19 [00:04<00:37,  2.20s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on contrast_5: 28.65999937057495%\n",
            "Processing defocus_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 16%|█▌        | 3/19 [00:06<00:31,  1.99s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on defocus_blur_5: 70.93999981880188%\n",
            "Processing elastic_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 21%|██        | 4/19 [00:08<00:28,  1.90s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on elastic_5: 72.43000268936157%\n",
            "Processing fog_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 26%|██▋       | 5/19 [00:09<00:25,  1.85s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on fog_5: 49.43999946117401%\n",
            "Processing frost_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 32%|███▏      | 6/19 [00:11<00:23,  1.81s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frost_5: 60.009998083114624%\n",
            "Processing frosted_glass_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 37%|███▋      | 7/19 [00:13<00:21,  1.79s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frosted_glass_blur_5: 58.139997720718384%\n",
            "Processing gaussian_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 42%|████▏     | 8/19 [00:15<00:19,  1.78s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_blur_5: 65.90999960899353%\n",
            "Processing gaussian_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 9/19 [00:16<00:17,  1.77s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_noise_5: 47.00999855995178%\n",
            "Processing impulse_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 53%|█████▎    | 10/19 [00:18<00:15,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on impulse_noise_5: 29.399999976158142%\n",
            "Processing jpeg_compression_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 58%|█████▊    | 11/19 [00:20<00:14,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on jpeg_compression_5: 76.10999941825867%\n",
            "Processing motion_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 63%|██████▎   | 12/19 [00:22<00:12,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on motion_blur_5: 65.93000292778015%\n",
            "Processing pixelate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 68%|██████▊   | 13/19 [00:23<00:10,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on pixelate_5: 71.3699996471405%\n",
            "Processing saturate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 74%|███████▎  | 14/19 [00:25<00:08,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on saturate_5: 80.47000169754028%\n",
            "Processing shot_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 79%|███████▉  | 15/19 [00:27<00:06,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on shot_noise_5: 51.42999887466431%\n",
            "Processing snow_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 84%|████████▍ | 16/19 [00:29<00:05,  1.74s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on snow_5: 66.38000011444092%\n",
            "Processing spatter_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 89%|████████▉ | 17/19 [00:30<00:03,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on spatter_5: 70.24000287055969%\n",
            "Processing speckle_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 95%|█████████▍| 18/19 [00:32<00:01,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on speckle_noise_5: 52.27000117301941%\n",
            "Processing zoom_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 19/19 [00:34<00:00,  1.81s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on zoom_blur_5: 77.46000289916992%\n",
            "Mean Top-1 Accuracy: 61.863684340527186%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "regional-composer"
      },
      "source": [
        "### Regular"
      ],
      "id": "regional-composer"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "valued-dominican",
        "outputId": "578da18e-4baa-4787-df05-36942085e974"
      },
      "source": [
        "# Evaluate teacher model \n",
        "teacher_model = get_training_model()\n",
        "teacher_model.load_weights(\"teacher_model.h5\")\n",
        "teacher_model.compile(loss=\"sparse_categorical_crossentropy\",\n",
        "                         metrics=[\"accuracy\"])\n",
        "acc_dict, mean_top_1 = evaluate_model(teacher_model)\n",
        "print(f\"Mean Top-1 Accuracy: {mean_top_1}%\")"
      ],
      "id": "valued-dominican",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 0/19 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing brightness_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  5%|▌         | 1/19 [00:03<00:55,  3.08s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on brightness_5: 75.52000284194946%\n",
            "Processing contrast_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 11%|█         | 2/19 [00:04<00:38,  2.29s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on contrast_5: 24.3599995970726%\n",
            "Processing defocus_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 16%|█▌        | 3/19 [00:06<00:32,  2.04s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on defocus_blur_5: 71.95000052452087%\n",
            "Processing elastic_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 21%|██        | 4/19 [00:08<00:28,  1.93s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on elastic_5: 74.55999851226807%\n",
            "Processing fog_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 26%|██▋       | 5/19 [00:10<00:26,  1.86s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on fog_5: 45.71999907493591%\n",
            "Processing frost_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 32%|███▏      | 6/19 [00:11<00:23,  1.82s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frost_5: 62.48999834060669%\n",
            "Processing frosted_glass_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 37%|███▋      | 7/19 [00:13<00:21,  1.80s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frosted_glass_blur_5: 63.33000063896179%\n",
            "Processing gaussian_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 42%|████▏     | 8/19 [00:15<00:19,  1.78s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_blur_5: 66.90000295639038%\n",
            "Processing gaussian_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 9/19 [00:17<00:17,  1.78s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_noise_5: 44.62999999523163%\n",
            "Processing impulse_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 53%|█████▎    | 10/19 [00:18<00:15,  1.77s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on impulse_noise_5: 27.889999747276306%\n",
            "Processing jpeg_compression_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 58%|█████▊    | 11/19 [00:20<00:14,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on jpeg_compression_5: 78.46999764442444%\n",
            "Processing motion_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 63%|██████▎   | 12/19 [00:22<00:12,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on motion_blur_5: 66.21999740600586%\n",
            "Processing pixelate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 68%|██████▊   | 13/19 [00:24<00:10,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on pixelate_5: 75.98999738693237%\n",
            "Processing saturate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 74%|███████▎  | 14/19 [00:25<00:08,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on saturate_5: 67.47999787330627%\n",
            "Processing shot_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 79%|███████▉  | 15/19 [00:27<00:06,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on shot_noise_5: 48.44000041484833%\n",
            "Processing snow_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 84%|████████▍ | 16/19 [00:29<00:05,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on snow_5: 65.85999727249146%\n",
            "Processing spatter_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 89%|████████▉ | 17/19 [00:31<00:03,  1.74s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on spatter_5: 63.81999850273132%\n",
            "Processing speckle_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 95%|█████████▍| 18/19 [00:32<00:01,  1.74s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on speckle_noise_5: 49.059998989105225%\n",
            "Processing zoom_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 19/19 [00:34<00:00,  1.82s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on zoom_blur_5: 77.02000141143799%\n",
            "Mean Top-1 Accuracy: 60.51105205949984%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hydraulic-finding",
        "outputId": "4b224c7f-5576-40b6-e3ad-cf26691793c8"
      },
      "source": [
        "# Evaluate the corresponding student model\n",
        "student_noisy = get_training_model()\n",
        "student_noisy.load_weights(\"student_noisy.h5\")\n",
        "student_noisy.compile(loss=\"sparse_categorical_crossentropy\",\n",
        "                         metrics=[\"accuracy\"])\n",
        "acc_dict, mean_top_1 = evaluate_model(student_noisy)\n",
        "print(f\"Mean Top-1 Accuracy: {mean_top_1}%\")"
      ],
      "id": "hydraulic-finding",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 0/19 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Processing brightness_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "  5%|▌         | 1/19 [00:02<00:52,  2.92s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on brightness_5: 79.80999946594238%\n",
            "Processing contrast_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 11%|█         | 2/19 [00:04<00:37,  2.23s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on contrast_5: 20.76999992132187%\n",
            "Processing defocus_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 16%|█▌        | 3/19 [00:06<00:32,  2.01s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on defocus_blur_5: 68.98000240325928%\n",
            "Processing elastic_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 21%|██        | 4/19 [00:08<00:28,  1.90s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on elastic_5: 71.53000235557556%\n",
            "Processing fog_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 26%|██▋       | 5/19 [00:09<00:25,  1.84s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on fog_5: 46.140000224113464%\n",
            "Processing frost_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 32%|███▏      | 6/19 [00:11<00:23,  1.81s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frost_5: 57.56999850273132%\n",
            "Processing frosted_glass_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 37%|███▋      | 7/19 [00:13<00:21,  1.79s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on frosted_glass_blur_5: 57.03999996185303%\n",
            "Processing gaussian_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 42%|████▏     | 8/19 [00:15<00:19,  1.78s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_blur_5: 63.429999351501465%\n",
            "Processing gaussian_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 47%|████▋     | 9/19 [00:16<00:17,  1.77s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on gaussian_noise_5: 44.06000077724457%\n",
            "Processing impulse_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 53%|█████▎    | 10/19 [00:18<00:15,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on impulse_noise_5: 24.68000054359436%\n",
            "Processing jpeg_compression_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 58%|█████▊    | 11/19 [00:20<00:14,  1.76s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on jpeg_compression_5: 74.55000281333923%\n",
            "Processing motion_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 63%|██████▎   | 12/19 [00:22<00:12,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on motion_blur_5: 62.459999322891235%\n",
            "Processing pixelate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 68%|██████▊   | 13/19 [00:23<00:10,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on pixelate_5: 70.16000151634216%\n",
            "Processing saturate_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 74%|███████▎  | 14/19 [00:25<00:08,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on saturate_5: 77.31000185012817%\n",
            "Processing shot_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 79%|███████▉  | 15/19 [00:27<00:07,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on shot_noise_5: 48.60000014305115%\n",
            "Processing snow_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 84%|████████▍ | 16/19 [00:29<00:05,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on snow_5: 63.2099986076355%\n",
            "Processing spatter_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 89%|████████▉ | 17/19 [00:30<00:03,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on spatter_5: 65.49000144004822%\n",
            "Processing speckle_noise_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " 95%|█████████▍| 18/19 [00:32<00:01,  1.75s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on speckle_noise_5: 48.96999895572662%\n",
            "Processing zoom_blur_5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 19/19 [00:34<00:00,  1.81s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy on zoom_blur_5: 75.85999965667725%\n",
            "Mean Top-1 Accuracy: 58.98000041120931%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "urban-sapphire"
      },
      "source": [
        "## Evaluate on CIFAR-10 Test Set"
      ],
      "id": "urban-sapphire"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "precious-release"
      },
      "source": [
        "(_, _), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()\n",
        "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
        "test_ds = prepare_dataset(test_ds)"
      ],
      "id": "precious-release",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "broadband-dream",
        "outputId": "cd48ea0c-e40d-41ae-9fa9-3ac681c5580f"
      },
      "source": [
        "# Evaluate teacher model trained with SWA\n",
        "_, test_acc = teacher_model_swa.evaluate(test_ds, verbose=0)\n",
        "print(\"Test accuracy with SWA teacher: {:.2f}%\".format(test_acc * 100))\n",
        "\n",
        "# Evaluate the corresponding student\n",
        "_, test_acc = student_noisy_swa.evaluate(test_ds, verbose=0)\n",
        "print(\"Test accuracy with noisy SWA student: {:.2f}%\".format(test_acc * 100))"
      ],
      "id": "broadband-dream",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy with SWA teacher: 84.82%\n",
            "Test accuracy with noisy SWA student: 85.24%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "enormous-repository",
        "outputId": "ae8923c8-7e55-4753-aa9e-2ee075bbef9f"
      },
      "source": [
        "# Evaluate teacher model trained with MA\n",
        "_, test_acc = teacher_model_ma.evaluate(test_ds, verbose=0)\n",
        "print(\"Test accuracy with MA teacher: {:.2f}%\".format(test_acc * 100))\n",
        "\n",
        "# Evaluate the corresponding student\n",
        "_, test_acc = student_noisy_ma.evaluate(test_ds, verbose=0)\n",
        "print(\"Test accuracy with noisy MA student: {:.2f}%\".format(test_acc * 100))"
      ],
      "id": "enormous-repository",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy with MA teacher: 83.88%\n",
            "Test accuracy with noisy MA student: 84.42%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dental-quest",
        "outputId": "3f18d6cc-9605-44ca-8001-9d9961cd499b"
      },
      "source": [
        "# Evaluate regular teacher model\n",
        "_, test_acc = teacher_model.evaluate(test_ds, verbose=0)\n",
        "print(\"Test accuracy with regular teacher: {:.2f}%\".format(test_acc * 100))\n",
        "\n",
        "# Evaluate the corresponding student\n",
        "_, test_acc = student_noisy.evaluate(test_ds, verbose=0)\n",
        "print(\"Test accuracy with regular noisy student: {:.2f}%\".format(test_acc * 100))"
      ],
      "id": "dental-quest",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy with regular teacher: 83.20%\n",
            "Test accuracy with regular noisy student: 82.16%\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}