housing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow.keras as keras\n",
    "import tensorflow as tf\n",
    "\n",
    "inputs = keras.Input((6))\n",
    "x = keras.layers.Dense(40, activation=\"selu\")(inputs)\n",
    "x = keras.layers.Dense(20, activation=\"selu\")(x)\n",
    "outputs = keras.layers.Dense(1, activation=\"relu\")(x)\n",
    "model = keras.Model(inputs, outputs)\n",
    "model.compile(loss=keras.losses.MeanSquaredError(), optimizer=keras.optimizers.Adam())\n",
    "\n",
    "intermediate_model = keras.Model(inputs, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "from sklearn import datasets\n",
    "\n",
    "x, y = datasets.fetch_california_housing(return_X_y=True)\n",
    "\n",
    "x = x[:, :6]\n",
    "\n",
    "base_x, base_y = x.copy(), y.copy()\n",
    "\n",
    "x_train, y_train, x_test, y_test = x[:10000], y[:10000], x[10000:], y[10000:]\n",
    "\n",
    "base_x_train, base_y_train, base_x_test, base_y_test = x_train.copy(), y_train.copy(), x_test.copy(), y_test.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# VERY coarse normalisation. Don't really care that much about performance on validation set\n",
    "# but still need to scale the data\n",
    "ZERO_MAX = x_train[:, 0].max()\n",
    "x_test[:, 0] /= ZERO_MAX\n",
    "x_train[:, 0] /= ZERO_MAX\n",
    "\n",
    "ONE_MAX = x_train[:, 1].max()\n",
    "\n",
    "x_test[:, 1] /= ONE_MAX\n",
    "x_train[:, 1] /= ONE_MAX\n",
    "\n",
    "x_test[:, 2] = np.clip(x_train[:, 2], 0, 20) / 20\n",
    "x_train[:, 2] = np.clip(x_train[:, 2], 0, 20) / 20\n",
    "\n",
    "x_test[:, 3] = np.clip(x_train[:, 3], 0, 3) / 3\n",
    "x_train[:, 3] = np.clip(x_train[:, 3], 0, 3) / 3\n",
    "\n",
    "x_test[:, 4] = np.log10(x_train[:, 4]) - 3\n",
    "x_train[:, 4] = np.log10(x_train[:, 4]) - 3\n",
    "\n",
    "x_test[:, 5] = np.clip(x_train[:, 5], 0, 8) / 8\n",
    "x_train[:, 5] = np.clip(x_train[:, 5], 0, 8) / 8\n",
    "\n",
    "# SIX_MAX = x_train[:, 6].max()\n",
    "\n",
    "# x_test[:, 6] = (x_test[:, 6] - 30) / SIX_MAX\n",
    "# x_train[:, 6] = (x_train[:, 6] - 30) / SIX_MAX\n",
    "\n",
    "# SEVEN_MIN = np.abs(x_train[:, 7].min())\n",
    "\n",
    "# x_test[:, 7] = (x_test[:, 7] + SEVEN_MIN) / 10\n",
    "# x_train[:, 7] = (x_train[:, 7] + SEVEN_MIN) / 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reverse_encoding(case):\n",
    "    case = case.copy()\n",
    "    case[0] *= ZERO_MAX\n",
    "    case[1] *= ONE_MAX\n",
    "    case[2] *= 20\n",
    "    case[3] *= 3\n",
    "    case[4] = 10 ** (case[4] + 3)\n",
    "    case[5] *= 8\n",
    "    # case[6] = case[6] * SIX_MAX + 30\n",
    "    # case[7] = case[7] * 10 - SEVEN_MIN\n",
    "\n",
    "    return case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(x_train, y_train, epochs=12, validation_data=(x_test, y_test), batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sb\n",
    "sb.histplot(x_test[:, 3] / x_test[:, 5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratio = x_test[:, 3] / x_test[:, 5]\n",
    "\n",
    "concept_index = ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "probe_input = keras.Input(20)\n",
    "probe_output = keras.layers.Dense(1, activity_regularizer=keras.regularizers.L1(0.01), activation=\"relu\")(probe_input)\n",
    "\n",
    "probe = keras.Model(probe_input, probe_output)\n",
    "probe.compile(loss=keras.losses.MeanAbsoluteError(), optimizer=keras.optimizers.Adam())\n",
    "\n",
    "inter_x_test = intermediate_model(x_test)\n",
    "probe_x_train, probe_y_train, probe_x_test, probe_y_test = inter_x_test[:2000], concept_index[:2000], inter_x_test[2000:], concept_index[2000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "probe.fit(probe_x_train, probe_y_train, validation_data=(probe_x_test, probe_y_test), epochs=17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_inputs = keras.Input(6)\n",
    "maximiser_input = keras.Input(1)\n",
    "max_plane = keras.layers.Dense(6, name=\"max_plane\", activity_regularizer=keras.regularizers.L1(0.25), activation=\"linear\")(maximiser_input)\n",
    "additive = keras.layers.Add()([max_plane, main_inputs])\n",
    "\n",
    "intermediate_model.trainable = False\n",
    "inter_outputs = intermediate_model(additive)\n",
    "\n",
    "maximised_probe_output = keras.layers.Dense(1, name=\"max_probe_repl\", activation=\"relu\")(inter_outputs)\n",
    "maximiser_model = keras.Model([main_inputs, maximiser_input], maximised_probe_output)\n",
    "look_at_results_model = keras.Model([main_inputs, maximiser_input], additive)\n",
    "max_plane_model = keras.Model([main_inputs, maximiser_input], max_plane)\n",
    "\n",
    "maximiser_model.get_layer(\"max_probe_repl\").set_weights(probe.layers[-1].get_weights())\n",
    "maximiser_model.get_layer(\"max_probe_repl\").trainable = False\n",
    "maximiser_model.get_layer(\"max_plane\").trainable = True\n",
    "\n",
    "maximiser_model.compile(loss=keras.losses.MeanSquaredError(), optimizer=keras.optimizers.Adam())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.argpartition(concept_index, 5)[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CASE_INDEX = 10248"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concept_index[CASE_INDEX]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "case = np.repeat(np.expand_dims(x_test[CASE_INDEX], axis=0), 64, axis=0)\n",
    "ones = np.expand_dims(np.ones(64), -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "maximiser_model.fit([case, ones], ones, epochs=300, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "maximiser_model([case, ones])[0].numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "maximised = look_at_results_model([case, ones])[0].numpy()\n",
    "reverse_encoding(maximised)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_x_test[CASE_INDEX]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_x_test[2649, 3] / base_x_test[2649, 5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reverse_encoding(maximised)[3] / reverse_encoding(maximised)[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.makedirs(\"housing_results\", exist_ok=True)\n",
    "\n",
    "np.savez(\"housing_results/{}.npz\".format(CASE_INDEX), original_case=base_x_test[CASE_INDEX], maximised_case=reverse_encoding(maximised))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.load(\"housing_results/{}.npz\".format(CASE_INDEX))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"original_case\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"maximised_case\"]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}