remove backslash, update file splitting based on os and sentiment ana…

…lyssi

.gitignore

@@ -30,3 +30,4 @@ tweets_dataset_gpu.csv
 tweets_dataset_all.csv
 json_files.txt
 combined_dataset.csv
+sentiment_evolution.ipynb

file_splitting/split_files.py

@@ -8,6 +8,11 @@
 data = [Path("data/"+file) for file in os.listdir('data')]
 data.sort(key=lambda x: x.name)
 
-with open('json_files.txt', 'w') as f:
-    f.write('\n'.join(str(file).replace('data/', '') for file in data))
+if os.name == 'nt':  # Windows
+    with open('json_files.txt', 'w') as f:
+        f.write('\n'.join(str(file).replace('data\\', '') for file in data))
+else:  # Other operating systems
+    with open('json_files.txt', 'w') as f:
+        f.write('\n'.join(str(file).replace('data/', '') for file in data))
+
 print("Files written to json_files.txt.")

preprocess/old_tweet_loader.py

@@ -141,6 +141,8 @@ def text_transformer(text):
     # text = re.sub(r'[^A-Za-z ]', '', text) # remove special characters
     text = re.sub(r'\n', '', text)
     text = re.sub(r'[,.!?]', '', text)
+    text = re.sub(r"\'", "", text)
+    text = re.sub(r'\\', '', text)
     text = text.strip()
     text = text.lower()
     return text

preprocess/preprocessor.py

@@ -69,6 +69,8 @@ def text_transformer(text):
     # text = re.sub(r'[^A-Za-z ]', '', text) # remove special characters
     text = re.sub(r'\n', '', text)
     text = re.sub(r'[,.!?]', '', text)
+    text = re.sub(r"\'", "", text)
+    text = re.sub(r'\\', '', text)
     text = text.strip()
     text = text.lower()
     return text

preprocess/test_loading_gpu.py

@@ -169,6 +169,8 @@ def text_transformer(text):
     # text = re.sub(r'[^A-Za-z ]', '', text) # remove special characters
     text = re.sub(r'\n', '', text)
     text = re.sub(r'[,.!?]', '', text)
+    text = re.sub(r"\'", "", text)
+    text = re.sub(r'\\', '', text)
     text = text.strip()
     text = text.lower()
     return text

preprocess/tweets_gpu_testing.ipynb

@@ -256,6 +256,8 @@
     "    # text = re.sub(r'[^A-Za-z ]', '', text) # remove special characters\n",
     "    text = re.sub(r'\\n', '', text)\n",
     "    text = re.sub(r'[,.!?]', '', text)\n",
+    "    text = re.sub(r\"\\'\", \"\", text)\n",
+    "    text = re.sub(r'\\\\', '', text)\n",
     "    text = text.strip()\n",
     "    text = text.lower()\n",
     "    return text"

preprocess/ultimate_tweet_loader.py

@@ -141,6 +141,8 @@ def text_transformer(text):
     # text = re.sub(r'[^A-Za-z ]', '', text) # remove special characters
     text = re.sub(r'\n', '', text)
     text = re.sub(r'[,.!?]', '', text)
+    text = re.sub(r"\'", "", text)
+    text = re.sub(r'\\', '', text)
     text = text.strip()
     text = text.lower()
     return text

sentiment/sentiment_analysis.ipynb

@@ -26,7 +26,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -48,7 +48,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -84,7 +84,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 58,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {