-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathconvert_to_text_task1.py
182 lines (136 loc) · 7.52 KB
/
convert_to_text_task1.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
import os, sys, csv, re
import numpy as np
import pandas as pd
import string
from sklearn.model_selection import train_test_split
from nltk.tokenize import RegexpTokenizer
from sklearn.utils import shuffle
tokenizer = RegexpTokenizer(r'\w+')
data_dir_1_train_dev = "./RDoC_raw_data/RDoCTask/RDoCTask1TrainData/Combined_Batch/"
data_dir_1_test = "./RDoC_raw_data/RDoCTask/RDoCTask1TestData/Combined_Batch/"
data_output_dir_1_train_dev = "./datasets/Task1_without_acronym/"
data_output_dir_1_test = "./datasets/Task1_test_data_without_acronym/"
FLOAT_REGEXP = re.compile(r'^[-+]?([0-9]+|[0-9]*\.[0-9]+)([eE][-+]?[0-9]+)?$')
CAPS_REMOVE_LIST = ["APPROACH", "BACKGROUND", "COMPARISON", "CONCLUSION", "CONCLUSIONS", "FINDINGS", "IMPLICATIONS",
"INTRODUCTION", "LIMITATIONS", "MEASURES", "METHODOLOGY", "METHOD", "METHODS", "OBJECTIVE",
"OBJECTIVES", "OUTCOME", "PURPOSE", "RESULTS", "RESEARCH", "SIGNIFICANCE", "STATEMENT", "STUDY", "SUMMARY"]
def is_float(str):
return True if FLOAT_REGEXP.match(str) else False
def replace_num(tokens):
new_tokens = []
for token in tokens:
if is_float(token):
new_tokens.append("<num>")
else:
new_tokens.append(token)
return new_tokens
def preprocess_token(token):
return "".join([char for char in token if char.isalpha()])
def remove_punctuation_and_replace_num(doc):
doc_tokens = [token.strip(string.punctuation) for token in tokenizer.tokenize(doc)]
doc_tokens = [token for token in doc_tokens if not token.isupper()]
#doc_tokens = [token for token in doc_tokens if not token in CAPS_REMOVE_LIST]
doc_tokens = replace_num(doc_tokens)
doc_tokens = [preprocess_token(token.lower()) for token in doc_tokens]
return " ".join(doc_tokens)
## Train/Dev data split and pre-processing
RDoC_contruct_fnames = [fname.split(".")[0] for fname in os.listdir(data_dir_1_train_dev) if fname.endswith(".xlsx")]
total_docs = {RDoC_name:[] for RDoC_name in RDoC_contruct_fnames}
total_titles = {RDoC_name:[] for RDoC_name in RDoC_contruct_fnames}
total_labels = {RDoC_name:[] for RDoC_name in RDoC_contruct_fnames}
total_ids = {RDoC_name:[] for RDoC_name in RDoC_contruct_fnames}
for RDoC_name in RDoC_contruct_fnames:
xl = pd.read_excel(open(data_dir_1_train_dev + RDoC_name + ".xlsx", 'rb'))
total_ids[RDoC_name].extend(xl['pmid'].tolist())
total_titles[RDoC_name].extend(xl['title'].tolist())
total_docs[RDoC_name].extend(xl['abstract'].tolist())
total_labels[RDoC_name].extend([RDoC_name] * len(total_docs[RDoC_name]))
train_docs = []
train_titles = []
train_labels = []
train_ids = []
val_docs = []
val_titles = []
val_labels = []
val_ids = []
for RDoC_name in RDoC_contruct_fnames:
docs_train, docs_val, labels_train, labels_val, titles_train, titles_val, ids_train, ids_val \
= train_test_split(total_docs[RDoC_name], total_labels[RDoC_name], total_titles[RDoC_name], total_ids[RDoC_name], test_size=0.2, random_state=123)
train_docs.extend(docs_train)
train_titles.extend(titles_train)
train_labels.extend(labels_train)
train_ids.extend(ids_train)
val_docs.extend(docs_val)
val_titles.extend(titles_val)
val_labels.extend(labels_val)
val_ids.extend(ids_val)
with open(data_dir_1_train_dev + "train_docs.txt", "w") as f:
for id, label, title, doc in zip(train_ids, train_labels, train_titles, train_docs):
f.write(str(id) + "<<>>" + label + "<<>>" + title + "<<>>" + doc + "\n")
with open(data_dir_1_train_dev + "val_docs.txt", "w") as f:
for id, label, title, doc in zip(val_ids, val_labels, val_titles, val_docs):
f.write(str(id) + "<<>>" + label + "<<>>" + title + "<<>>" + doc + "\n")
train_docs = [remove_punctuation_and_replace_num(doc) for doc in train_docs]
val_docs = [remove_punctuation_and_replace_num(doc) for doc in val_docs]
train_titles = [remove_punctuation_and_replace_num(doc) for doc in train_titles]
val_titles = [remove_punctuation_and_replace_num(doc) for doc in val_titles]
train_docs, train_titles, train_labels = shuffle(train_docs, train_titles, train_labels, random_state=123)
val_docs, val_titles, val_labels = shuffle(val_docs, val_titles, val_labels, random_state=123)
if not os.path.exists(data_output_dir_1_train_dev):
os.makedirs(data_output_dir_1_train_dev)
with open(data_output_dir_1_train_dev + "train_docs.txt", "w") as f:
f.write("\n".join([label + "\t" + title + " " + doc for doc, title, label in zip(train_docs, train_titles, train_labels)]))
with open(data_output_dir_1_train_dev + "train_ids.txt", "w") as f:
f.write("\n".join([str(id) for id in train_ids]))
with open(data_output_dir_1_train_dev + "val_docs.txt", "w") as f:
f.write("\n".join([label + "\t" + title + " " + doc for doc, title, label in zip(val_docs, val_titles, val_labels)]))
with open(data_output_dir_1_train_dev + "val_ids.txt", "w") as f:
f.write("\n".join([str(id) for id in val_ids]))
## Test data pre-processing
RDoC_contruct_fnames_test = [fname.split(".")[0] for fname in os.listdir(data_dir_1_test) if fname.endswith(".xlsx")]
total_docs = {RDoC_name:[] for RDoC_name in RDoC_contruct_fnames_test}
total_titles = {RDoC_name:[] for RDoC_name in RDoC_contruct_fnames_test}
total_labels = {RDoC_name:[] for RDoC_name in RDoC_contruct_fnames_test}
total_ids = {RDoC_name:[] for RDoC_name in RDoC_contruct_fnames_test}
with open(data_dir_1_test + "test_docs.txt", "w") as f:
pass
for RDoC_name in RDoC_contruct_fnames_test:
xl = pd.read_excel(open(data_dir_1_test + RDoC_name + ".xlsx", 'rb'))
total_ids[RDoC_name].extend(xl['pmid'].tolist())
total_titles[RDoC_name].extend(xl['title'].tolist())
total_docs[RDoC_name].extend(xl['abstract'].tolist())
total_labels[RDoC_name].extend([RDoC_name] * len(total_docs[RDoC_name]))
with open(data_dir_1_test + "test_docs.txt", "a") as f:
for id, label, title, doc in zip(total_ids[RDoC_name], total_labels[RDoC_name], total_titles[RDoC_name], total_docs[RDoC_name]):
f.write(str(id) + "<<>>" + label + "<<>>" + title + "<<>>" + doc + "\n")
test_docs = []
test_titles = []
test_labels = []
test_ids = []
with open(data_dir_1_test + "test_docs.txt", "r") as f:
for line in f.readlines():
id, label, title, doc = line.strip().split("<<>>")
test_docs.append(doc)
test_titles.append(title)
test_labels.append(label)
test_ids.append(id)
test_docs = [remove_punctuation_and_replace_num(doc) for doc in test_docs]
test_titles = [remove_punctuation_and_replace_num(doc) for doc in test_titles]
new_test_labels = []
for label in test_labels:
if label == "Potential_Threat_Anxiety":
new_test_labels.append("Potential_Threat_Anxiety_")
else:
new_test_labels.append(label)
if not os.path.exists(data_output_dir_1_test):
os.makedirs(data_output_dir_1_test)
with open(data_output_dir_1_test + "test_docs.txt", "w") as f:
#f.write("\n".join([label + "\t" + title + " " + doc for doc, title, label in zip(test_docs, test_titles, test_labels)]))
f.write("\n".join([label + "\t" + title + " " + doc for doc, title, label in zip(test_docs, test_titles, new_test_labels)]))
with open(data_output_dir_1_test + "test_ids.txt", "w") as f:
f.write("\n".join([str(id) for id in test_ids]))
with open(data_output_dir_1_train_dev + "test_docs.txt", "w") as f:
#f.write("\n".join([label + "\t" + title + " " + doc for doc, title, label in zip(test_docs, test_titles, test_labels)]))
f.write("\n".join([label + "\t" + title + " " + doc for doc, title, label in zip(test_docs, test_titles, new_test_labels)]))
with open(data_output_dir_1_train_dev + "test_ids.txt", "w") as f:
f.write("\n".join([str(id) for id in test_ids]))