MST698S_Session6_NetworkX_Build_Social_Graph.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "colab": {
      "name": "MST698S_Session6_NetworkX_Build_Social_Graph.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dVBPo8JnXymo",
        "colab_type": "text"
      },
      "source": [
        "## MST 698S - Data Science Tools and Techniques\n",
        "\n",
        "## Network X - Build Network from Twitter Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wkYZ3gQqj23y",
        "colab_type": "text"
      },
      "source": [
        "Note that it may be faster to run this on a GPU."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iBMviIBvXynF",
        "colab_type": "text"
      },
      "source": [
        "Import relevant libraries:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gMbk_c1PXynU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import networkx as nx\n",
        "import json\n",
        "import numpy as np\n",
        "from copy import deepcopy\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2Sh64MYsXynt",
        "colab_type": "text"
      },
      "source": [
        "Define global parameters:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pCTrrAbWXyn0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "twitter_followers_path = r'twitter_followers.json'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PMccZHntXyoJ",
        "colab_type": "text"
      },
      "source": [
        "Define relevant functions:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CTZuS8X8XyoO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Get set of user_ids\n",
        "def get_unique_ids(followers_dict_list):\n",
        "    all_screen_names_list = []\n",
        "    for follower in followers_dict_list:\n",
        "        followers_list = list(follower.values())[0]\n",
        "        all_screen_names_list = all_screen_names_list + followers_list\n",
        "        all_screen_names_list.append(list(follower.keys())[0])\n",
        "    unique_screen_names = list(set(all_screen_names_list))\n",
        "    return unique_screen_names\n",
        "\n",
        "#Get unique edges\n",
        "def clean_edge_list(edge_list):\n",
        "    temp = []\n",
        "    for (a,b) in edge_list:\n",
        "        if (a,b) not in temp and (b,a) not in temp:\n",
        "            if (a,b) != (b,a):\n",
        "                temp.append((a,b))\n",
        "    output = 1*temp\n",
        "    return output\n",
        "\n",
        "#Get edge list\n",
        "def get_edges(followers_dict_list):\n",
        "    edge_list = []\n",
        "    edge_list_out = []\n",
        "    for relationship in followers_dict_list:\n",
        "        for friend in list(relationship.values())[0]:\n",
        "            edge = (list(relationship.keys())[0],friend)\n",
        "            edge_list.append(edge)\n",
        "    edge_list = list(set(edge_list))\n",
        "    edge_list_out = clean_edge_list(edge_list)\n",
        "    return edge_list_out\n",
        "\n",
        "#build graph\n",
        "def build_ego_graph(unique_screen_names, edge_list_out):\n",
        "    G = nx.Graph()\n",
        "    G.add_nodes_from(unique_screen_names)\n",
        "    G.add_edges_from(edge_list_out)\n",
        "    return G"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7h95zp-1Xyof",
        "colab_type": "text"
      },
      "source": [
        "Let's import our twitter followers list and print the first few items:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "r1BvId9ZXyoi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open(twitter_followers_path, 'r') as twitter_file:\n",
        "    followers_list = json.load(twitter_file)\n",
        "\n",
        "print(followers_list[:10])\n",
        "print(len(followers_list))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rYnnfZ8LXyoy",
        "colab_type": "text"
      },
      "source": [
        "Now lets clean up the followers list, extract the useful information, and build our graph object (note that some of these processes may take a while...):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gdBkeom-Xyo7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Get unique user ids:\n",
        "unique_screen_names = get_unique_ids(followers_list)\n",
        "\n",
        "#Get connections (edges)\n",
        "edge_list_out = get_edges(followers_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N-lKCV80g2oo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "f'Total Unique users: {str(len(unique_screen_names))}'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dGd4S9Pehq1C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "f'Total Graph Edges: {str(len(edge_list_out))}'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "69dmfVk6bN4e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Instantiate graph object\n",
        "G = build_ego_graph(unique_screen_names, edge_list_out)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bqqb4qiDXypo",
        "colab_type": "text"
      },
      "source": [
        "Now, let's prune the graph to remove screen names with only one connection to aid in visualiation:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uVNLgUEQXypu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Calculate degree centralities\n",
        "degree_centralities = nx.degree_centrality(G)\n",
        "one_degree = min(degree_centralities.values())\n",
        "\n",
        "#Get nodes of one degree:\n",
        "one_degree_nodes = []\n",
        "for node in degree_centralities.keys():\n",
        "    if degree_centralities[node] == one_degree:\n",
        "        one_degree_nodes.append(node)\n",
        "\n",
        "#Create copy of the graph \n",
        "H = deepcopy(G)\n",
        "\n",
        "#Remove nodes with only one connection\n",
        "for node in one_degree_nodes:\n",
        "    H.remove_node(node)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80EptatBh5U4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "f'Total nodes before pruning: {str(len(G.nodes()))}'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lO8D7ZeAh5iF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "f'Total nodes after pruning: {str(len(H.nodes()))}'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vm3opprRXyqc",
        "colab_type": "text"
      },
      "source": [
        "Now let's use the betweenness centralities to find the 10 most influential nodes (this may take a while...):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vZX1CUyCXyqk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Function to sort the list by first item of tuple \n",
        "def Sort_Tuple(tup):  \n",
        "    # key is set to sort using first element  \n",
        "    # sublist lambda has been used  \n",
        "    tup.sort(key = lambda x: x[0])\n",
        "    tup.reverse()  \n",
        "    return tup\n",
        "\n",
        "#Calculate centralities\n",
        "betweenness_centralities = nx.betweenness_centrality(H)\n",
        "\n",
        "#Print node with maximum betweenness centrality\n",
        "inverse = [(value, key) for key, value in betweenness_centralities.items()]\n",
        "inverse_sort = Sort_Tuple(inverse)\n",
        "inverse_sort[:10]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PmfTHKX8Xyp-",
        "colab_type": "text"
      },
      "source": [
        "Now let's draw the pruned graph (this may take a while...):"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ofAyqmkEXyqC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nx.draw(H)\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}