Test_reddit_api_SM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bd483961",
   "metadata": {},
   "source": [
    "### REDDIT API Connexion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1131a33e",
   "metadata": {},
   "outputs": [],
   "source": [
    "CLIENT_ID = ''\n",
    "SECRET = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "00cafe82",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "import csv\n",
    "import time\n",
    "from datetime import datetime\n",
    "import os\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9e5b4f98",
   "metadata": {},
   "outputs": [],
   "source": [
    "auth = requests.auth.HTTPBasicAuth(CLIENT_ID, SECRET)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c865fb73",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {\n",
    "    'grant_type' : 'password', \n",
    "    'username' : '', \n",
    "    'password' : '' \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4e41f049",
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {'User-Agent': 'MyAPI/0.0.1'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c13ec447",
   "metadata": {},
   "outputs": [],
   "source": [
    "res  = requests.post('https://www.reddit.com/api/v1/access_token', \n",
    "                    auth=auth, data=data, headers = headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "66df7f99",
   "metadata": {},
   "outputs": [],
   "source": [
    "TOKEN = res.json()['access_token']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0ac72958",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'eyJhbGciOiJSUzI1NiIsImtpZCI6IlNIQTI1NjpzS3dsMnlsV0VtMjVmcXhwTU40cWY4MXE2OWFFdWFyMnpLMUdhVGxjdWNZIiwidHlwIjoiSldUIn0.eyJzdWIiOiJ1c2VyIiwiZXhwIjoxNzAyNDk5OTI0LjkxMTI4NCwiaWF0IjoxNzAyNDEzNTI0LjkxMTI4NCwianRpIjoiN3hYNk1zZVZmcEhwY2k4TFdqY2Y5aVJwN3Bzd1BBIiwiY2lkIjoiUnQ3ZDI4OWJPbXFkNEFyQWpwUXd3ZyIsImxpZCI6InQyX25lbnh0a3lmZyIsImFpZCI6InQyX25lbnh0a3lmZyIsImxjYSI6MTY5OTM3MjYzNjcyNCwic2NwIjoiZUp5S1Z0SlNpZ1VFQUFEX193TnpBU2MiLCJmbG8iOjl9.sf8pCVsg7egnSoBA0TVnwc6Iurg7xdgjWuK3Y3vFyZeek8EFBA7JtFgVKRVogrFU97c3BlK18Gr2OaVxZuO4gAvnA_YR1Q2RNmYB2ZCTZo70mnVbsDWWmwBXlyxCBKrvxUWmR_xZoJTaxiGXfgzEqwseqDsiqcOLLiYWTaV9Ler8cwrPlk2qWsOG0zePtxKIO1xRAdlL6WZ-ekAj-PUCvLAw-0fYH_UvHkI_XhPhHrtzXsqLfoswHlrtkMw88Fobsof1i7HJlTjIcqyQ0D8LF2v1hzyJrOYBtRHQvV5CCv1CJKArJpopSJCJJu70_TiIzWOPv8pjUisUuQ0Vpd7bEg'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TOKEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "eae69618",
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {**headers, **{'Authorization': f'bearer {TOKEN}'}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "013df73d",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'User-Agent': 'MyAPI/0.0.1',\n",
       " 'Authorization': 'bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IlNIQTI1NjpzS3dsMnlsV0VtMjVmcXhwTU40cWY4MXE2OWFFdWFyMnpLMUdhVGxjdWNZIiwidHlwIjoiSldUIn0.eyJzdWIiOiJ1c2VyIiwiZXhwIjoxNzAyNDk5OTI0LjkxMTI4NCwiaWF0IjoxNzAyNDEzNTI0LjkxMTI4NCwianRpIjoiN3hYNk1zZVZmcEhwY2k4TFdqY2Y5aVJwN3Bzd1BBIiwiY2lkIjoiUnQ3ZDI4OWJPbXFkNEFyQWpwUXd3ZyIsImxpZCI6InQyX25lbnh0a3lmZyIsImFpZCI6InQyX25lbnh0a3lmZyIsImxjYSI6MTY5OTM3MjYzNjcyNCwic2NwIjoiZUp5S1Z0SlNpZ1VFQUFEX193TnpBU2MiLCJmbG8iOjl9.sf8pCVsg7egnSoBA0TVnwc6Iurg7xdgjWuK3Y3vFyZeek8EFBA7JtFgVKRVogrFU97c3BlK18Gr2OaVxZuO4gAvnA_YR1Q2RNmYB2ZCTZo70mnVbsDWWmwBXlyxCBKrvxUWmR_xZoJTaxiGXfgzEqwseqDsiqcOLLiYWTaV9Ler8cwrPlk2qWsOG0zePtxKIO1xRAdlL6WZ-ekAj-PUCvLAw-0fYH_UvHkI_XhPhHrtzXsqLfoswHlrtkMw88Fobsof1i7HJlTjIcqyQ0D8LF2v1hzyJrOYBtRHQvV5CCv1CJKArJpopSJCJJu70_TiIzWOPv8pjUisUuQ0Vpd7bEg'}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0548cab",
   "metadata": {},
   "source": [
    "## Récupération des comptes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f783181c",
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'limit': 15000,  # Limite le nombre de résultats à 100 par page\n",
    "    'after': None  # Permet de récupérer les résultats suivants\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3ecba0b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def recupAccounts(subName):\n",
    "    i = 0\n",
    "    df = pd.DataFrame()\n",
    "    url = f'https://oauth.reddit.com/r/{subName}/hot'\n",
    "    while True:\n",
    "        response = requests.get(url, headers=headers, params=params)\n",
    "        i +=1\n",
    "        print(i)\n",
    "        if i == 1000:\n",
    "            break\n",
    "        if response.status_code == 200:\n",
    "            data = response.json()\n",
    "            for post in data['data']['children']:\n",
    "                #df = df.append({'author':post['data']['author']}, ignore_index=True)\n",
    "                new_row = pd.DataFrame({'author': [post['data']['author']]})\n",
    "\n",
    "                df = pd.concat([df, new_row], ignore_index=True)\n",
    "\n",
    "            \n",
    "            # Mettez à jour le paramètre 'after' pour obtenir les résultats suivants\n",
    "            params['after'] = data['data']['after']\n",
    "\n",
    "\n",
    "            # Arrêtez la boucle si 'after' est None, ce qui signifie qu'il n'y a plus de résultats à récupérer\n",
    "            if params['after'] is None:\n",
    "                break\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "TopSubReddit = pd.read_csv('C:/Users/samma/Desktop/ultimatelist.csv') \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c0365bf",
   "metadata": {},
   "source": [
    "### Récupération des subReddit d'un compte"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4c6af980",
   "metadata": {},
   "outputs": [],
   "source": [
    "def recupSubReddit(username):\n",
    "    dfSub = pd.DataFrame()\n",
    "    after = None\n",
    "    headers = {'User-Agent': 'MyAPI/0.0.1'}  # Assurez-vous d'avoir un User-Agent valide\n",
    "\n",
    "    while True:\n",
    "        urlDos = f'https://www.reddit.com/user/{username}/submitted.json'\n",
    "        params = {'after': after} if after else {}\n",
    "        response = requests.get(urlDos, headers=headers, params=params)\n",
    "\n",
    "        if response.status_code == 200:\n",
    "            submissions = response.json()['data']['children']\n",
    "            for submission in submissions:\n",
    "                new_row = pd.DataFrame({username: [submission['data']['subreddit']]}, index=[0])\n",
    "                dfSub = pd.concat([dfSub, new_row], ignore_index=True)\n",
    "\n",
    "            after = response.json()['data']['after']\n",
    "\n",
    "            if not after:\n",
    "                break\n",
    "        elif response.status_code == 404:\n",
    "            print(f\"Utilisateur {username} non trouvé.\")\n",
    "            break\n",
    "        elif response.status_code == 429:\n",
    "            reset_time = int(response.headers['x-ratelimit-reset'])\n",
    "            current_time = int(time.time())\n",
    "\n",
    "            # Calcule le temps d'attente jusqu'à la réinitialisation du taux limite\n",
    "            wait_time = max(0, reset_time - current_time) + 20  # Ajoute un délai supplémentaire de 5 secondes\n",
    "\n",
    "            print(f\"Erreur 429 - Trop de requêtes. Attendez {wait_time} secondes avant de réessayer.\")\n",
    "            time.sleep(wait_time)\n",
    "        else:\n",
    "            print(f\"La requête a échoué avec le code d'état {response.status_code}\")\n",
    "            break\n",
    "            \n",
    "    return dfSub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recupAutoSub(acounts):\n",
    "    dfs = []\n",
    "    i = 0\n",
    "    for compte in acounts.itertuples(index=False):\n",
    "        print(i)\n",
    "\n",
    "        i += 1\n",
    "        username = compte.author\n",
    "        if username != '[deleted]' and username:  \n",
    "            df_sub = recupSubReddit(username).drop_duplicates()\n",
    "            dfs.append(df_sub)\n",
    "\n",
    "    df_result = pd.concat(dfs, axis=1)\n",
    "    return df_result\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mainFonct(nbSub):\n",
    "    for i in range(12, nbSub):\n",
    "        print(i)\n",
    "        #sub_name = TopSubReddit['announcements'][i]\n",
    "        sub_name = 'france'\n",
    "        print(sub_name)\n",
    "        \n",
    "        # Appel de la fonction recupAccounts\n",
    "        df_accounts = recupAccounts(sub_name)\n",
    "        \n",
    "        # Affichage du DataFrame\n",
    "        print(df_accounts)\n",
    "        \n",
    "        # Affichage de la forme du DataFrame\n",
    "        print(df_accounts.shape)\n",
    "\n",
    "        maintenant = datetime.now()\n",
    "        date_heure_lisible = maintenant.strftime(\"%d-%m-%Y %H:%M:%S\")\n",
    "        print(f\"\\n\\n\\nDébut de {sub_name}\\n {date_heure_lisible}\\n\\n\\n\")\n",
    "        \n",
    "        df_result = recupAutoSub(df_accounts)\n",
    "        df_result.to_csv(f'CSV/{TopSubReddit[\"announcements\"][i]}', index=False)\n",
    "\n",
    "        maintenant = datetime.now()\n",
    "        date_heure_lisible = maintenant.strftime(\"%d-%m-%Y %H:%M:%S\")\n",
    "        print(f\"\\n\\n\\Fin de {sub_name}\\n {date_heure_lisible}\\n\\n\\n\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n",
      "france\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "                   author\n",
      "0           AutoModerator\n",
      "1           AutoModerator\n",
      "2                  Najbox\n",
      "3                 Taletad\n",
      "4    InternationalDiet631\n",
      "..                    ...\n",
      "879                zoxume\n",
      "880               guilamu\n",
      "881           ultrajambon\n",
      "882               imre-gz\n",
      "883              shira_13\n",
      "\n",
      "[884 rows x 1 columns]\n",
      "(884, 1)\n",
      "\n",
      "\n",
      "\n",
      "Début de france\n",
      " 12-12-2023 21:40:16\n",
      "\n",
      "\n",
      "\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "mainFonct(13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "990bbaf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def liensCsv(fichier, tabUsers):\n",
    "    df = pd.read_csv(fichier)\n",
    "    nouveau_df = pd.DataFrame(columns=['Utilisateur', 'Subreddit'])\n",
    "    for colonne in df.columns:\n",
    "        if len(df[colonne]) > 1 and ('.' not in colonne):\n",
    "            subreddits = [subreddit for subreddit in df[colonne][1:] if pd.notna(subreddit)]\n",
    "            if subreddits:\n",
    "                paires = pd.DataFrame({'Utilisateur': [colonne] * len(subreddits), 'Subreddit': subreddits})\n",
    "                nouveau_df = pd.concat([nouveau_df, paires], ignore_index=True)\n",
    "        #if colonne not in tabUsers:\n",
    "            #tabUsers.append(colonne)\n",
    "\n",
    "    return (nouveau_df, tabUsers)      \n",
    "    #nouveau_df.to_csv('nouveau_fichier.csv', index=False)\n",
    "    # (colonne[:-2] not in tabUsers) and (colonne[:-3] not in tabUsers)\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "liensCsv('CSV/space')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def noeudsCsv(fichier, subList):\n",
    "    df = pd.read_csv(fichier)\n",
    "    nouveau_df = pd.DataFrame(columns=['ID', 'Label', 'Type'])\n",
    "    for colonne in df.columns:\n",
    "        if len(df[colonne]) > 1 and ('.' not in colonne):\n",
    "            # Username\n",
    "            triplet = pd.DataFrame({'ID': [colonne], 'Label': None, 'Type' : 'Utilisateur'})\n",
    "            nouveau_df = pd.concat([nouveau_df, triplet], ignore_index=True)\n",
    "\n",
    "            for ligne in df[colonne]:\n",
    "                if ligne not in subList and pd.notna(ligne):\n",
    "                    triplet = pd.DataFrame({'ID': [ligne], 'Label': [ligne], 'Type' : 'SubReddit'})\n",
    "                    nouveau_df = pd.concat([nouveau_df, triplet], ignore_index=True)\n",
    "                    \n",
    "                    subList.append(ligne)\n",
    "\n",
    "\n",
    "    return (nouveau_df, subList)   \n",
    "    #nouveau_df.to_csv('noeuds.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mainCSVFinal ():\n",
    "    noeuds = pd.DataFrame(columns=['ID', 'Label', 'Type'])\n",
    "    liens = pd.DataFrame(columns=['Utilisateur', 'Subreddit'])\n",
    "\n",
    "    repertoire_csv = \"CSV/\"\n",
    "\n",
    "    #listSub = ['CSV/Music', 'CSV/news']\n",
    "    fichiers_csv = [f\"CSV/{f}\" for f in os.listdir(repertoire_csv)]\n",
    "\n",
    "    subList = []\n",
    "    tabUsers = []\n",
    "    i = 1\n",
    "    for sub in fichiers_csv:\n",
    "        print(i)\n",
    "        print(sub)\n",
    "        i+=1\n",
    "\n",
    "\n",
    "        resLiens = liensCsv(sub, tabUsers)\n",
    "        liens = pd.concat([liens, resLiens[0]], ignore_index=True)\n",
    "        tabUsers = resLiens[1]\n",
    "\n",
    "        resNoeuds = noeudsCsv(sub, subList)\n",
    "        noeuds = pd.concat([noeuds, resNoeuds[0]], ignore_index=True)\n",
    "        subList = resNoeuds[1]\n",
    "\n",
    "        if i == 6:\n",
    "            break\n",
    "\n",
    "    liens.to_csv('liensTest2.csv', index=False)\n",
    "    noeuds.to_csv('noeudsTest2.csv', index=False)\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "0d9eeaaa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "CSV/4chan\n",
      "2\n",
      "CSV/AdviceAnimals\n",
      "3\n",
      "CSV/Android\n",
      "4\n",
      "CSV/AnimalsBeingJerks\n",
      "5\n",
      "CSV/Art\n",
      "6\n",
      "CSV/AskReddit\n",
      "7\n",
      "CSV/askscience\n",
      "8\n",
      "CSV/atheism\n",
      "9\n",
      "CSV/aww\n",
      "10\n",
      "CSV/bestof\n",
      "11\n",
      "CSV/BlackPeopleTwitter\n",
      "12\n",
      "CSV/blog\n",
      "13\n",
      "CSV/books\n",
      "14\n",
      "CSV/buildapc\n",
      "15\n",
      "CSV/comics\n",
      "16\n",
      "CSV/CrappyDesign\n",
      "17\n",
      "CSV/creepy\n",
      "18\n",
      "CSV/cringepics\n",
      "19\n",
      "CSV/dankmemes\n",
      "20\n",
      "CSV/dataisbeautiful\n",
      "21\n",
      "CSV/DIY\n",
      "22\n",
      "CSV/Documentaries\n",
      "23\n",
      "CSV/EarthPorn\n",
      "24\n",
      "CSV/europe\n",
      "25\n",
      "CSV/explainlikeimfive\n",
      "26\n",
      "CSV/facepalm\n",
      "27\n",
      "CSV/Fitness\n",
      "28\n",
      "CSV/food\n",
      "29\n",
      "CSV/FoodPorn\n",
      "30\n",
      "CSV/france\n",
      "31\n",
      "CSV/funny\n",
      "32\n",
      "CSV/Futurology\n",
      "33\n",
      "CSV/gadgets\n",
      "34\n",
      "CSV/gameofthrones\n",
      "35\n",
      "CSV/Games\n",
      "36\n",
      "CSV/gaming\n",
      "37\n",
      "CSV/GetMotivated\n",
      "38\n",
      "CSV/gifs\n",
      "39\n",
      "CSV/gonewild\n",
      "40\n",
      "CSV/history\n",
      "41\n",
      "CSV/HistoryPorn\n",
      "42\n",
      "CSV/IAmA\n",
      "43\n",
      "CSV/instant_regret\n",
      "44\n",
      "CSV/interestingasfuck\n",
      "45\n",
      "CSV/InternetIsBeautiful\n",
      "46\n",
      "CSV/Jokes\n",
      "47\n",
      "CSV/leagueoflegends\n",
      "48\n",
      "CSV/lifehacks\n",
      "49\n",
      "CSV/LifeProTips\n",
      "50\n",
      "CSV/listentothis\n",
      "51\n",
      "CSV/malefashionadvice\n",
      "52\n",
      "CSV/memes\n",
      "53\n",
      "CSV/me_irl\n",
      "54\n",
      "CSV/mildlyinfuriating\n",
      "55\n",
      "CSV/mildlyinteresting\n",
      "56\n",
      "CSV/movies\n",
      "57\n",
      "CSV/Music\n",
      "58\n",
      "CSV/NatureIsFuckingLit\n",
      "59\n",
      "CSV/nba\n",
      "60\n",
      "CSV/news\n",
      "61\n",
      "CSV/nosleep\n",
      "62\n",
      "CSV/nsfw\n",
      "63\n",
      "CSV/NSFW_GIF\n",
      "64\n",
      "CSV/oddlysatisfying\n",
      "65\n",
      "CSV/OldSchoolCool\n",
      "66\n",
      "CSV/Overwatch\n",
      "67\n",
      "CSV/pcmasterrace\n",
      "68\n",
      "CSV/personalfinance\n",
      "69\n",
      "CSV/philosophy\n",
      "70\n",
      "CSV/photoshopbattles\n",
      "71\n",
      "CSV/pics\n",
      "72\n",
      "CSV/pokemon\n",
      "73\n",
      "CSV/politics\n",
      "74\n",
      "CSV/programming\n",
      "75\n",
      "CSV/reactiongifs\n",
      "76\n",
      "CSV/relationships\n",
      "77\n",
      "CSV/science\n",
      "78\n",
      "CSV/sex\n",
      "79\n",
      "CSV/Showerthoughts\n",
      "80\n",
      "CSV/soccer\n",
      "81\n",
      "CSV/space\n",
      "82\n",
      "CSV/sports\n",
      "83\n",
      "CSV/tattoos\n",
      "84\n",
      "CSV/technology\n",
      "85\n",
      "CSV/television\n",
      "86\n",
      "CSV/tifu\n",
      "87\n",
      "CSV/Tinder\n",
      "88\n",
      "CSV/todayilearned\n",
      "89\n",
      "CSV/travel\n",
      "90\n",
      "CSV/trees\n",
      "91\n",
      "CSV/TwoXChromosomes\n",
      "92\n",
      "CSV/Unexpected\n",
      "93\n",
      "CSV/UpliftingNews\n",
      "94\n",
      "CSV/videos\n",
      "95\n",
      "CSV/Whatcouldgowrong\n",
      "96\n",
      "CSV/wholesomememes\n",
      "97\n",
      "CSV/woahdude\n",
      "98\n",
      "CSV/worldnews\n",
      "99\n",
      "CSV/WritingPrompts\n",
      "100\n",
      "CSV/WTF\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "def liensCsv(fichier, tabUsers, unique_links):\n",
    "    df = pd.read_csv(fichier)\n",
    "    nouveau_df = pd.DataFrame(columns=['Utilisateur', 'Subreddit'])\n",
    "\n",
    "    for colonne in df.columns:\n",
    "        if len(df[colonne]) > 1 and ('.' not in colonne) and (colonne != \"AutoModerator\"):\n",
    "            subreddits = [subreddit for subreddit in df[colonne][1:] if pd.notna(subreddit)]\n",
    "            if subreddits:\n",
    "                paires = pd.DataFrame({'Utilisateur': colonne, 'Subreddit': subreddits})\n",
    "                # Ensure only unique pairs are added\n",
    "                unique_pairs = list(zip(paires['Utilisateur'], paires['Subreddit']))\n",
    "                unique_pairs = [pair for pair in unique_pairs if pair not in unique_links]\n",
    "                unique_links.update(unique_pairs)\n",
    "                paires = pd.DataFrame(unique_pairs, columns=['Utilisateur', 'Subreddit'])\n",
    "                nouveau_df = pd.concat([nouveau_df, paires], ignore_index=True)\n",
    "\n",
    "        if colonne not in tabUsers:\n",
    "            tabUsers.add(colonne)\n",
    "\n",
    "    return (nouveau_df, tabUsers, unique_links)\n",
    "\n",
    "# Rest of the code remains unchanged...\n",
    "\n",
    "\n",
    "def noeudsCsv(fichier, subList):\n",
    "    df = pd.read_csv(fichier)\n",
    "    nouveau_df = pd.DataFrame(columns=['ID', 'Label', 'Type'])\n",
    "\n",
    "    for colonne in df.columns:\n",
    "        if len(df[colonne]) > 1 and ('.' not in colonne) and (colonne != \"AutoModerator\"):\n",
    "            triplet = pd.DataFrame({'ID': [colonne], 'Label': [None], 'Type': ['Utilisateur']})\n",
    "            nouveau_df = pd.concat([nouveau_df, triplet], ignore_index=True)\n",
    "\n",
    "            for ligne in df[colonne]:\n",
    "                if ligne not in subList and pd.notna(ligne):\n",
    "                    triplet = pd.DataFrame({'ID': [ligne], 'Label': [ligne], 'Type': ['SubReddit']})\n",
    "                    nouveau_df = pd.concat([nouveau_df, triplet], ignore_index=True)\n",
    "                    subList.add(ligne)\n",
    "\n",
    "    return (nouveau_df, subList)\n",
    "\n",
    "def mainCSVFinalOpti():\n",
    "    noeuds_list = []  # List to accumulate 'noeuds' DataFrames\n",
    "    liens_list = []   # List to accumulate 'liens' DataFrames\n",
    "    unique_links = set()  # Set to store unique pairs of 'Utilisateur' and 'Subreddit'\n",
    "\n",
    "    repertoire_csv = \"CSV/\"\n",
    "    fichiers_csv = [f\"CSV/{f}\" for f in os.listdir(repertoire_csv)]\n",
    "\n",
    "    subList = set()  # Use a set for faster membership tests\n",
    "    tabUsers = set()  # Use a set for faster membership tests\n",
    "    i = 1\n",
    "\n",
    "    for sub in fichiers_csv:\n",
    "        print(i)\n",
    "        print(sub)\n",
    "        i += 1\n",
    "\n",
    "        resLiens = liensCsv(sub, tabUsers, unique_links)\n",
    "        liens_list.append(resLiens[0])  # Append to the list instead of concatenating\n",
    "        tabUsers.update(resLiens[1])\n",
    "        unique_links = resLiens[2]\n",
    "\n",
    "        resNoeuds = noeudsCsv(sub, subList)\n",
    "        noeuds_list.append(resNoeuds[0])  # Append to the list instead of concatenating\n",
    "        subList.update(resNoeuds[1])\n",
    "\n",
    "\n",
    "    liens = pd.concat(liens_list, ignore_index=True)  # Concatenate the 'liens' DataFrames outside the loop\n",
    "\n",
    "    # Remove duplicate rows from the 'liens' DataFrame\n",
    "    liens = liens.drop_duplicates()\n",
    "    liens.to_csv('OptiliensTest2.csv', index=False)\n",
    "\n",
    "    noeuds = pd.concat(noeuds_list, ignore_index=True)  # Concatenate the 'noeuds' DataFrames outside the loop\n",
    "    noeuds = noeuds.drop_duplicates()\n",
    "\n",
    "    noeuds.to_csv('OptinoeudsTest2.csv', index=False)\n",
    "\n",
    "# Run the mainCSVFinalOpti function\n",
    "mainCSVFinalOpti()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "82f79a36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "repertoire_csv = \"CSV/\"\n",
    "fichiers_csv = [f\"CSV/{f}\" for f in os.listdir(repertoire_csv)]\n",
    "len(fichiers_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a05a1dd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "CSV/AdviceAnimals\n",
      "2\n",
      "CSV/Android\n",
      "3\n",
      "CSV/Art\n",
      "4\n",
      "CSV/AskReddit\n",
      "5\n",
      "CSV/askscience\n",
      "6\n",
      "CSV/atheism\n",
      "7\n",
      "CSV/aww\n",
      "8\n",
      "CSV/bestof\n",
      "9\n",
      "CSV/BlackPeopleTwitter\n",
      "10\n",
      "CSV/books\n",
      "11\n",
      "CSV/CrappyDesign\n",
      "12\n",
      "CSV/creepy\n",
      "13\n",
      "CSV/dankmemes\n",
      "14\n",
      "CSV/dataisbeautiful\n"
     ]
    }
   ],
   "source": [
    "mainCSVFinalOpti()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bf4d2282",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def nettoyage(nbmax):\n",
    "    # Charger le DataFrame initial\n",
    "    df = pd.read_csv(\"OptiliensTest2.csv\")\n",
    "    dfNoeuds = pd.read_csv(\"OptinoeudsTest2.csv\")\n",
    "\n",
    "    # Charger la liste des 3000 premiers Subreddits\n",
    "    TopSubReddit = pd.read_csv(\"ultimatelist.csv\", header=None, names=[\"id\", \"Subreddit\", \"nsfw\"])\n",
    "\n",
    "    # Filtrer les lignes en fonction du Top 3000\n",
    "    df_nettoyerliens = df[df['Subreddit'].isin(TopSubReddit['Subreddit'][500:nbmax])]\n",
    "\n",
    "    # Récupérer les utilisateurs présents dans les liens avec \"id\" comme nom d'utilisateur\n",
    "    utilisateurs_liens = pd.DataFrame({'ID': df_nettoyerliens['Utilisateur'], 'Label': '', 'Type': 'Utilisateur'})\n",
    "\n",
    "    noeuds_subnbmax = pd.DataFrame({'ID': df_nettoyerliens['Subreddit'], 'Label': df_nettoyerliens['Subreddit'], 'Type': 'Subreddit'})\n",
    "\n",
    "    # Concaténer les nœuds initiaux avec les nouveaux utilisateurs\n",
    "    df_nettoyerNoeuds = pd.DataFrame()\n",
    "    df_nettoyerNoeuds = pd.concat([noeuds_subnbmax, utilisateurs_liens], ignore_index=True)\n",
    "\n",
    "\n",
    "    # Supprimer les doublons pour s'assurer que les utilisateurs sont uniques dans les nœuds\n",
    "    df_nettoyerNoeuds = df_nettoyerNoeuds.drop_duplicates(subset='ID')\n",
    "\n",
    "    # Sauvegarder les nouveaux nœuds\n",
    "    df_nettoyerNoeuds.to_csv('500to3000Noeuds.csv', index=False)\n",
    "\n",
    "    # Sauvegarder le nouveau DataFrame de liens\n",
    "    df_nettoyerliens.to_csv('500to3000liens.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c352c4d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "nettoyage(3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c9473007",
   "metadata": {},
   "outputs": [],
   "source": [
    "nettoyage(500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def liensCsv(fichier, tabUsers):\n",
    "    df = pd.read_csv(fichier)\n",
    "    nouveau_df = pd.DataFrame(columns=['Utilisateur', 'Subreddit'])\n",
    "    for colonne in df.columns:\n",
    "        if len(df[colonne]) > 1 and (colonne[:-2] not in tabUsers['id']):\n",
    "            subreddits = [subreddit for subreddit in df[colonne][1:] if pd.notna(subreddit)]\n",
    "            if subreddits:\n",
    "                paires = pd.DataFrame({'Utilisateur': [colonne] * len(subreddits), 'Subreddit': subreddits})\n",
    "                nouveau_df = pd.concat([nouveau_df, paires], ignore_index=True)\n",
    "\n",
    "\n",
    "    return (nouveau_df)      \n",
    "    #nouveau_df.to_csv('nouveau_fichier.csv', index=False)\n",
    "    \n",
    "\n",
    "def noeudsCsv(fichier, dfres):\n",
    "    df = pd.read_csv(fichier)\n",
    "    nouveau_df = pd.DataFrame(columns=['ID', 'Label', 'Type'])\n",
    "    for colonne in df.columns:\n",
    "        if len(df[colonne]) > 1 and ('.' not in colonne ):\n",
    "\n",
    "            # Username\n",
    "            triplet = pd.DataFrame({'ID': [colonne], 'Label': None, 'Type' : 'Utilisateur'})\n",
    "            nouveau_df = pd.concat([nouveau_df, triplet], ignore_index=True)\n",
    "\n",
    "    for colonne in df.columns:\n",
    "            for ligne in df[colonne]:\n",
    "                if ~dfres['ID'].isin([colonne]).any() and (dfres['Type'] == 'SubReddit').any() and pd.notna(ligne):\n",
    "                    triplet = pd.DataFrame({'ID': [ligne], 'Label': [ligne], 'Type' : 'SubReddit'})\n",
    "                    nouveau_df = pd.concat([nouveau_df, triplet], ignore_index=True)\n",
    "\n",
    "\n",
    "\n",
    "    return (nouveau_df)   \n",
    "    #nouveau_df.to_csv('noeuds.csv', index=False)\n",
    "\n",
    "\n",
    "def mainCSVFinal3 ():\n",
    "    noeuds = pd.DataFrame(columns=['ID', 'Label', 'Type'])\n",
    "    liens = pd.DataFrame(columns=['Utilisateur', 'Subreddit'])\n",
    "\n",
    "    repertoire_csv = \"CSV/\"\n",
    "\n",
    "    #listSub = ['CSV/Music', 'CSV/news']\n",
    "    fichiers_csv = [f\"CSV/{f}\" for f in os.listdir(repertoire_csv)]\n",
    "\n",
    "    i = 1\n",
    "    for sub in fichiers_csv:\n",
    "        print(i)\n",
    "        i+=1\n",
    "\n",
    "        resNoeuds = noeudsCsv(sub, noeuds)\n",
    "        noeuds = pd.concat([noeuds, resNoeuds[0]], ignore_index=True)\n",
    "\n",
    "        resLiens = liensCsv(sub, noeuds)\n",
    "        liens = pd.concat([liens, resLiens[0]], ignore_index=True)\n",
    "\n",
    "\n",
    "\n",
    "        if i == 5:\n",
    "            break\n",
    "\n",
    "    liens.to_csv('liensTest3.csv', index=False)\n",
    "    noeuds.to_csv('noeudsTest3.csv', index=False)\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80bd8b7e",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "def noeudsCsv(fichier, subList):\n",
    "    df = pd.read_csv(fichier)\n",
    "    nouveau_df = pd.DataFrame(columns=['Type', 'ID', 'Label'])\n",
    "\n",
    "    for colonne in df.columns:\n",
    "        if len(df[colonne]) > 1:\n",
    "            # Username\n",
    "            triplet = pd.DataFrame([['Utilisateur', colonne, None]], columns=['Type', 'ID', 'Label'])\n",
    "            nouveau_df = nouveau_df.append(triplet, ignore_index=True)\n",
    "\n",
    "    mask = ~df.isin(subList) & df.notna()\n",
    "    for colonne in df.columns[mask.any()]:\n",
    "        for ligne in df[colonne][mask[colonne]]:\n",
    "            triplet = pd.DataFrame([['SubReddit', ligne, ligne]], columns=['Type', 'ID', 'Label'])\n",
    "            nouveau_df = nouveau_df.append(triplet, ignore_index=True)\n",
    "            subList.append(ligne)\n",
    "\n",
    "    return (nouveau_df, subList)\n",
    "\n",
    "def liensCsv(fichier, tabUsers):\n",
    "    df = pd.read_csv(fichier)\n",
    "    nouveau_df = pd.DataFrame(columns=['Utilisateur', 'Subreddit'])\n",
    "\n",
    "    for colonne in df.columns[df.columns.isin(tabUsers)]:\n",
    "        if len(df[colonne]) > 1:\n",
    "            subreddits = [subreddit for subreddit in df[colonne][1:] if pd.notna(subreddit)]\n",
    "            if subreddits:\n",
    "                paires = pd.DataFrame({'Utilisateur': [colonne] * len(subreddits), 'Subreddit': subreddits})\n",
    "                nouveau_df = nouveau_df.append(paires, ignore_index=True)\n",
    "        tabUsers.append(colonne)\n",
    "\n",
    "    return (nouveau_df, tabUsers)\n",
    "\n",
    "def mainCSVFinal():\n",
    "    noeuds = pd.DataFrame(columns=['Type', 'ID', 'Label'])\n",
    "    liens = pd.DataFrame(columns=['Utilisateur', 'Subreddit'])\n",
    "\n",
    "    repertoire_csv = \"CSV/\"\n",
    "    fichiers_csv = [f\"CSV/{f}\" for f in os.listdir(repertoire_csv)]\n",
    "\n",
    "    subList = []\n",
    "    tabUsers = []\n",
    "    i = 1\n",
    "    for sub in fichiers_csv:\n",
    "        print(i)\n",
    "        i += 1\n",
    "        resNoeuds = noeudsCsv(sub, subList)\n",
    "        noeuds = noeuds.append(resNoeuds[0], ignore_index=True)\n",
    "        subList = resNoeuds[1]\n",
    "\n",
    "        resLiens = liensCsv(sub, tabUsers)\n",
    "        liens = liens.append(resLiens[0], ignore_index=True)\n",
    "        tabUsers = resLiens[1]\n",
    "\n",
    "        if i == 5:\n",
    "            break\n",
    "\n",
    "    liens.to_csv('liensTest.csv', index=False)\n",
    "    noeuds.to_csv('noeudsTest.csv', index=False)\n",
    "\n",
    "mainCSVFinal()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}