ReproducibilityProvenance.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ReproducibilityProvenance.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyM/WpS3nr4ekwolILqvg1R3"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N26tYM8skDaE"
      },
      "source": [
        "# Pré-Requisitos"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RkktDws8uRby"
      },
      "source": [
        "## PROV - https://pypi.org/project/prov/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J6Jy23dZuVPO"
      },
      "source": [
        "!pip install prov"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7hKEstIKt4ga"
      },
      "source": [
        "## Sweetviz - https://pypi.org/project/sweetviz/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FxSjhW8wkHai"
      },
      "source": [
        "!pip install sweetviz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZUJ7D_1miI3f"
      },
      "source": [
        "# Bibliotecas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "97aNGhY-hDWJ"
      },
      "source": [
        "import sweetviz as sv\n",
        "import pandas as pd\n",
        "import prov\n",
        "import bs4"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JAxVjNn8woun"
      },
      "source": [
        "# Reprodutibilidade"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7fC9_a3_i7Yp"
      },
      "source": [
        "## Mostra as versões das bibliotecas usadas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eF36i0fOiMyC"
      },
      "source": [
        "import sys\n",
        "\n",
        "print('Versão do Pandas:', pd.__version__)\n",
        "print('Versão do Beautiful Soup:', bs4.__version__)\n",
        "print('Versão do PROV:', prov.__version__)\n",
        "print('Versão do Sweetviz:', sv.__version__)\n",
        "print('Versão do Python:', sys.version)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "71rc-9M2rsv9"
      },
      "source": [
        "## Sistema Operacional"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Yj11vhdprZH"
      },
      "source": [
        "!cat /etc/os-release"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wua_ucxmsVRJ"
      },
      "source": [
        "### Versão do kernel"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dl7XhFABsBjT"
      },
      "source": [
        "!cat /proc/version"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ENh1FZ-aJKgT"
      },
      "source": [
        "### Arquitetura (32 ou 64 bits)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z4KyKtfQJRTB"
      },
      "source": [
        "!lscpu"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wuraH9qKyFYN"
      },
      "source": [
        "# Provenance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mFPVVXrQyb1z"
      },
      "source": [
        "from prov import model\n",
        "# Create a new provenance document\n",
        "doc = model.ProvDocument()\n",
        "# Declaring namespaces\n",
        "doc.set_default_namespace('https://github.com/ppgi-ufrj-data-science/FakeNews')\n",
        "doc.add_namespace('kagg', 'https://www.kaggle.com/')\n",
        "doc.add_namespace('ppgi', 'http://www.ppgi.ufrj.br/')\n",
        "# Criando entidades\n",
        "dataset = doc.entity('kagg:fake-news', {model.PROV_TYPE : 'kagg:dataset', model.PROV_LOCATION : 'https://www.kaggle.com/mrisdal/fake-news',\n",
        "                                        'kagg:title' : 'Getting Real about Fake News', 'kagg:resume' : 'Text & metadata from fake & biased news sources around the web',\n",
        "                                        'kagg:usability' : '7.1', 'kagg:license' : 'CC0: Public Domain', 'kagg:tags' : 'arts and entertainment, news, politics, languages',\n",
        "                                        'kagg:records' : '12.999', 'kagg:columns' : '20'})\n",
        "zip = doc.entity('kagg:FakeNewData.zip', {model.PROV_TYPE : 'File', 'kagg:size' : '19.4 MB'})\n",
        "csv = doc.entity('kagg:fake.csv', {model.PROV_TYPE : 'File', 'kagg:size' : '54.05 MB'})\n",
        "clean_data = doc.entity('ppgi:Dataset com Dados Tratados')\n",
        "proc_data = doc.entity('ppgi:Dataset com Texto Processado')\n",
        "artigo = doc.entity('ppgi:Artigo sobre Análise Exploratória de Dados - Fake News')\n",
        "# Criando agentes\n",
        "meg = doc.agent('kagg:Meg Risdal', {model.PROV_TYPE : 'prov:Person', 'kagg:position': 'Product Manager', model.PROV_LOCATION : 'Los Angeles, California, United States',\n",
        "                                    'kagg:user' : 'mrisdal', 'kagg:profile' : 'https://www.kaggle.com/mrisdal'})\n",
        "bsdet = doc.agent('B.S. Detector', {model.PROV_TYPE : 'prov:SoftwareAgent', 'prov:publisher' : 'The Self Agency, LLC', 'prov:creator' : 'Daniel Sieradski',\n",
        "                                    model.PROV_LOCATION : 'https://addons.mozilla.org/en-US/firefox/addon/bsdetector/'})\n",
        "api = doc.agent('webhose.io API', {model.PROV_TYPE : 'prov:SoftwareAgent', 'prov:publisher' : 'Webhose', model.PROV_LOCATION : 'https://webhose.io/'})\n",
        "sirius = doc.agent('ppgi:Sírius Thadeu', {model.PROV_TYPE : 'prov:Person', model.PROV_ROLE : 'Aluno', 'prov:organization' : 'PPGI-UFRJ',\n",
        "                                          model.PROV_LOCATION : 'Rio de Janeiro - RJ'})\n",
        "# Criando atividades\n",
        "scrap = doc.activity('kagg:Scrap Text and Metadata', '25/10/2016', '25/11/2016', {'kagg:source' : '244 websites'})\n",
        "data_clean = doc.activity('ppgi:Limpeza de Dados')\n",
        "text_proc = doc.activity('ppgi:Processamento de Texto')\n",
        "eda = doc.activity('ppgi:Análise Exploratória de Dados')\n",
        "# Relacionamentos\n",
        "doc.used(data_clean, dataset)\n",
        "doc.used(text_proc, clean_data)\n",
        "doc.used(eda, proc_data)\n",
        "doc.wasInfluencedBy(scrap, api)\n",
        "doc.wasInfluencedBy(scrap, bsdet)\n",
        "scrap.wasAssociatedWith(meg)\n",
        "data_clean.wasAssociatedWith(sirius)\n",
        "text_proc.wasAssociatedWith(sirius)\n",
        "eda.wasAssociatedWith(sirius)\n",
        "dataset.wasAttributedTo(meg)\n",
        "dataset.wasGeneratedBy(scrap)\n",
        "zip.wasDerivedFrom(dataset)\n",
        "csv.wasDerivedFrom(zip)\n",
        "clean_data.wasGeneratedBy(data_clean)\n",
        "clean_data.wasDerivedFrom(csv)\n",
        "proc_data.wasGeneratedBy(text_proc)\n",
        "proc_data.wasDerivedFrom(clean_data)\n",
        "artigo.wasGeneratedBy(eda)\n",
        "artigo.wasDerivedFrom(proc_data)\n",
        "artigo.wasAttributedTo(sirius)\n",
        "print(doc.get_provn())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "agvaqzZWSh4j"
      },
      "source": [
        "## Gera o gráfico"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FNNiF6PgSwyp"
      },
      "source": [
        "from prov import dot\n",
        "# Save the provenance graph\n",
        "dot = dot.prov_to_dot(doc)\n",
        "dot.write_png('fake-news-dataset-prov.png')\n",
        "# Visualize the graph file\n",
        "from IPython.display import Image\n",
        "Image('fake-news-dataset-prov.png')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}