-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathkmeans.py
140 lines (106 loc) · 4.34 KB
/
kmeans.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
import numpy as np
import pandas as pd
import nltk
import re
import os
import codecs
from sklearn import feature_extraction
import random
stopwords = nltk.corpus.stopwords.words('english')
#print stopwords[:10]
'''import sys
reload(sys)
sys.setdefaultencoding('utf8')'''
from nltk.stem.snowball import SnowballStemmer
stemmer = SnowballStemmer("english")
text = open ("biggboss10","r")
synopses =[]
for line in text:
line=line.strip()
if line!='':
synopses.append(line)
#print synopses[0][:200]
#print len(synopses)
#json.loads(unicode(opener.open(...), "ISO-8859-1"))
def tokenize_and_stem(text):
# first tokenize by sentence, then by word to ensure that punctuation is caught as it's own token
tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
filtered_tokens = []
# filter out any tokens not containing letters (e.g., numeric tokens, raw punctuation)
for token in tokens:
if re.search('[a-zA-Z]', token):
filtered_tokens.append(token)
stems = [stemmer.stem(t) for t in filtered_tokens]
return stems
def tokenize_only(text):
# first tokenize by sentence, then by word to ensure that punctuation is caught as it's own token
tokens = [word.lower() for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
filtered_tokens = []
# filter out any tokens not containing letters (e.g., numeric tokens, raw punctuation)
for token in tokens:
if re.search('[a-zA-Z]', token):
filtered_tokens.append(token)
return filtered_tokens
'''with open ("biggboss10","r") as f:
text=f.read()
text= unicode(text, errors='ignore')
tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
# filter out any tokens not containing letters (e.g., numeric tokens, raw punctuation)
for token in tokens:
if re.search('[a-zA-Z]', token):
filtered_tokens1.append(token)
stems = [stemmer.stem(t) for t in filtered_tokens1]
filtered_tokens = []
with open ("biggboss10","r") as f:
text=f.read()
text= unicode(text, errors='ignore')
tokens = [word.lower() for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
for token in tokens:
if re.search('[a-zA-Z]', token):
filtered_tokens.append(token)'''
def main(tt):
#not super pythonic, no, not at all.
#use extend so it's a big flat list of vocab
totalvocab_stemmed = []
totalvocab_tokenized = []
for i in synopses:
i = i.decode('utf-8').replace(u'\u014c\u0106\u014d','-')
allwords_stemmed = tokenize_and_stem(i) #for each item in 'synopses', tokenize/stem
totalvocab_stemmed.extend(allwords_stemmed) #extend the 'totalvocab_stemmed' list
allwords_tokenized = tokenize_only(i)
totalvocab_tokenized.extend(allwords_tokenized)
vocab_frame = pd.DataFrame({'words': totalvocab_tokenized}, index = totalvocab_stemmed)
#print 'there are ' + str(vocab_frame.shape[0]) + ' items in vocab_frame'
#print totalvocab_stemmed[0]
'''print vocab_frame.head()
print
print
print
print'''
from sklearn.feature_extraction.text import TfidfVectorizer
#define vectorizer parameters
tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=200000,min_df=0.2, stop_words='english',use_idf=True, tokenizer=tokenize_and_stem, ngram_range=(1,3))
tfidf_matrix = tfidf_vectorizer.fit_transform(synopses) #fit the vectorizer to synopses
#idf_matrix=tfidf_vectorizer.inverse_transform(synopses)
#print(tfidf_matrix.shape)
terms = tfidf_vectorizer.get_feature_names()
#print type(tfidf_matrix)
from sklearn.metrics.pairwise import cosine_similarity
dist = 1 - cosine_similarity(tfidf_matrix)
from sklearn.cluster import KMeans
num_clusters = 13
km = KMeans(n_clusters=num_clusters)
km.fit(tfidf_matrix)
clusters = km.labels_.tolist()
from sklearn.externals import joblib
#uncomment the below to save your model
#since I've already run my model I am loading from the pickle
joblib.dump(km, 'doc_cluster.pkl')
km = joblib.load('doc_cluster.pkl')
clusters = km.labels_.tolist()
clus=random.randrange(0, 13)
strr = " "
for i in range(len(clusters)):
if (clusters[i]==clus):
strr += synopses[i]+"\n"
return strr