-
Notifications
You must be signed in to change notification settings - Fork 454
/
tweets_analyzer.py
executable file
·477 lines (388 loc) · 16.9 KB
/
tweets_analyzer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2017 @x0rz
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# Usage:
# python tweets_analyzer.py -n screen_name
#
# Install:
# pip install tweepy ascii_graph tqdm numpy
from __future__ import unicode_literals
from ascii_graph import Pyasciigraph
from ascii_graph.colors import Gre, Yel, Red
from ascii_graph.colordata import hcolor
from tqdm import tqdm
import tweepy
import numpy
import argparse
import collections
import datetime
import re
import json
import sys
import os
__version__ = '0.2-dev'
try:
from urllib.parse import urlparse
except ImportError:
from urlparse import urlparse
from secrets import consumer_key, consumer_secret, access_token, access_token_secret
# Here are sglobals used to store data - I know it's dirty, whatever
start_date = 0
end_date = 0
export = ""
jsono = {}
save_folder = "tweets"
color_supported = True
ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
parser = argparse.ArgumentParser(description=
"Simple Twitter Profile Analyzer (https://github.com/x0rz/tweets_analyzer) version %s" % __version__,
usage='%(prog)s -n <screen_name> [options]')
parser.add_argument('-l', '--limit', metavar='N', type=int, default=1000,
help='limit the number of tweets to retreive (default=1000)')
parser.add_argument('-n', '--name', required=True, metavar="screen_name",
help='target screen_name')
parser.add_argument('-f', '--filter', help='filter by source (ex. -f android will get android tweets only)')
parser.add_argument('--no-timezone', action='store_true',
help='removes the timezone auto-adjustment (default is UTC)')
parser.add_argument('--utc-offset', type=int,
help='manually apply a timezone offset (in seconds)')
parser.add_argument('--friends', action='store_true',
help='will perform quick friends analysis based on lang and timezone (rate limit = 15 requests)')
parser.add_argument('-e', '--export', metavar='path/to/file', type=str,
help='exports results to file')
parser.add_argument('-j', '--json', action='store_true',
help='outputs json')
parser.add_argument('-s', '--save', action='store_true',
help='saves tweets to %s/{twitter_handle}/{yyyy-mm-dd_HH-MM-SS}.json' %save_folder)
parser.add_argument('--no-color', action='store_true',
help='disables colored output')
parser.add_argument('--no-retweets', action='store_true',
help='does not evaluate retweets')
args = parser.parse_args()
activity_hourly = {
("%2i:00" % i).replace(" ", "0"): 0 for i in range(24)
}
activity_weekly = {
"%i" % i: 0 for i in range(7)
}
detected_langs = collections.Counter()
detected_sources = collections.Counter()
detected_places = collections.Counter()
geo_enabled_tweets = 0
detected_hashtags = collections.Counter()
detected_domains = collections.Counter()
detected_timezones = collections.Counter()
retweets = 0
retweeted_users = collections.Counter()
mentioned_users = collections.Counter()
id_screen_names = {}
friends_timezone = collections.Counter()
friends_lang = collections.Counter()
def process_tweet(tweet):
""" Processing a single Tweet and updating our datasets """
global start_date
global end_date
global geo_enabled_tweets
global retweets
if args.no_retweets:
if hasattr(tweet, 'retweeted_status'):
return
if hasattr(tweet, 'is_quote_status') and tweet.is_quote_status:
return
# Check for filters before processing any further
if args.filter and tweet.source:
if not args.filter.lower() in tweet.source.lower():
return
tw_date = tweet.created_at
# Updating most recent tweet
end_date = end_date or tw_date
start_date = tw_date
# Handling retweets
try:
# We use id to get unique accounts (screen_name can be changed)
rt_id_user = tweet.retweeted_status.user.id_str
retweeted_users[rt_id_user] += 1
if tweet.retweeted_status.user.screen_name not in id_screen_names:
id_screen_names[rt_id_user] = "@%s" % tweet.retweeted_status.user.screen_name
retweets += 1
except:
pass
# Adding timezone from profile offset to set to local hours
if tweet.user.utc_offset and not args.no_timezone:
tw_date = (tweet.created_at + datetime.timedelta(seconds=tweet.user.utc_offset))
if args.utc_offset:
tw_date = (tweet.created_at + datetime.timedelta(seconds=args.utc_offset))
# Updating our activity datasets (distribution maps)
activity_hourly["%s:00" % str(tw_date.hour).zfill(2)] += 1
activity_weekly[str(tw_date.weekday())] += 1
# Updating langs
detected_langs[tweet.lang] += 1
# Updating sources
detected_sources[tweet.source] += 1
# Detecting geolocation
if tweet.place:
geo_enabled_tweets += 1
tweet.place.name = tweet.place.name
detected_places[tweet.place.name] += 1
# Updating hashtags list
if tweet.entities['hashtags']:
for ht in tweet.entities['hashtags']:
ht['text'] = "#%s" % ht['text']
detected_hashtags[ht['text']] += 1
# Updating domains list
if tweet.entities['urls']:
for url in tweet.entities['urls']:
domain = urlparse(url['expanded_url']).netloc
if domain != "twitter.com": # removing twitter.com from domains (not very relevant)
detected_domains[domain] += 1
# Updating mentioned users list
if tweet.entities['user_mentions']:
for ht in tweet.entities['user_mentions']:
mentioned_users[ht['id_str']] += 1
if not ht['screen_name'] in id_screen_names:
id_screen_names[ht['id_str']] = "@%s" % ht['screen_name']
def process_friend(friend):
""" Process a single friend """
friends_lang[friend.lang] += 1 # Getting friend language & timezone
if friend.time_zone:
friends_timezone[friend.time_zone] += 1
def get_friends(api, username, limit):
""" Download friends and process them """
for friend in tqdm(tweepy.Cursor(api.friends, screen_name=username).items(limit), unit="friends", total=limit):
process_friend(friend)
def get_tweets(api, username, fh, limit):
""" Download Tweets from username account """
if args.json is False:
for status in tqdm(tweepy.Cursor(api.user_timeline, screen_name=username).items(limit), unit="tw", total=limit):
process_tweet(status)
if args.save:
fh.write(str(json.dumps(status._json))+",")
else:
for status in (tweepy.Cursor(api.user_timeline, screen_name=username).items(limit)):
process_tweet(status)
if args.save:
fh.write(str(json.dumps(status._json))+",")
def int_to_weekday(day):
weekdays = "Monday Tuesday Wednesday Thursday Friday Saturday Sunday".split()
return weekdays[int(day) % len(weekdays)]
def supports_color():
if args.no_color:
return False
# copied from https://github.com/django/django/blob/master/django/core/management/color.py
plat = sys.platform
supported_platform = plat != 'Pocket PC' and (plat != 'win32' or 'ANSICON' in os.environ)
# isatty is not always implemented, #6223.
is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
if not supported_platform or not is_a_tty:
return False
return True
def cprint(strng):
if not color_supported:
strng = ansi_escape.sub('', strng)
if args.json is False:
print(strng)
export_string(strng)
def export_string(strng):
global export
if args.export is not None:
export+=strng+"\n"
def export_write():
global export
if args.export is not None:
text_file = open(args.export, "w")
if args.json is False:
# remove ANSI color codes for export
export = ansi_escape.sub('', export)
# remove non ascii characters
export = "".join(i for i in export if ord(i)<128)
else:
export = json.dumps(jsono)
text_file.write(export)
text_file.close()
def print_stats(dataset, top=5):
""" Displays top values by order """
sum = numpy.sum(list(dataset.values()))
i = 0
if sum:
sorted_keys = sorted(dataset, key=dataset.get, reverse=True)
max_len_key = max([len(x) for x in sorted_keys][:top]) # use to adjust column width
for k in sorted_keys:
try:
cprint(("- \033[1m{:<%d}\033[0m {:>6} {:<4}" % max_len_key)
.format(k, dataset[k], "(%d%%)" % ((float(dataset[k]) / sum) * 100)))
except:
import ipdb
ipdb.set_trace()
i += 1
if i >= top:
break
else:
cprint("No data")
cprint("")
def print_charts(dataset, title, weekday=False):
""" Prints nice charts based on a dict {(key, value), ...} """
chart = []
keys = sorted(dataset.keys())
mean = numpy.mean(list(dataset.values()))
median = numpy.median(list(dataset.values()))
if args.json is False:
export_string(title)
for key in keys:
if (dataset[key] >= median * 1.33):
displayed_key = "%s (\033[92m+\033[0m)" % (int_to_weekday(key) if weekday else key)
elif (dataset[key] <= median * 0.66):
displayed_key = "%s (\033[91m-\033[0m)" % (int_to_weekday(key) if weekday else key)
else:
displayed_key = (int_to_weekday(key) if weekday else key)
if args.json is False:
export_string("%s - %s" % (dataset[key], (int_to_weekday(key) if weekday else key)))
chart.append((displayed_key, dataset[key]))
thresholds = {
int(mean): Gre, int(mean * 2): Yel, int(mean * 3): Red,
}
data = hcolor(chart, thresholds)
graph = Pyasciigraph(
separator_length=4,
multivalue=False,
human_readable='si',
)
if args.json is False:
for line in graph.graph(title, data):
if not color_supported:
ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
line = ansi_escape.sub('', line)
print(line)
cprint("")
def main():
global color_supported
color_supported = supports_color()
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
twitter_api = tweepy.API(auth)
now = datetime.datetime.now()
save_path = save_folder+"/"+args.name
save_file = False
if args.save:
if not os.path.exists(save_path):
os.makedirs(save_path)
save_file = open(save_path+"/"+now.strftime("%Y-%m-%d_%H-%M-%S")+".json","w")
save_file.write("[")
# Getting general account's metadata
cprint("[+] Getting @%s account data..." % args.name)
jsono['user_name'] = args.name
user_info = twitter_api.get_user(screen_name=args.name)
cprint("[+] lang : \033[1m%s\033[0m" % user_info.lang)
cprint("[+] geo_enabled : \033[1m%s\033[0m" % user_info.geo_enabled)
cprint("[+] time_zone : \033[1m%s\033[0m" % user_info.time_zone)
cprint("[+] utc_offset : \033[1m%s\033[0m" % user_info.utc_offset)
jsono['user_lang'] = user_info.lang
jsono['user_geo_enabled'] = user_info.geo_enabled
jsono['user_time_zone'] = user_info.time_zone
jsono['user_utc_offset'] = user_info.utc_offset
if user_info.utc_offset is None:
cprint("[\033[91m!\033[0m] Can't get specific timezone for this user")
jsono['user_utc_offset_note'] = "Can't get specific timezone for this user"
if args.utc_offset:
cprint("[\033[91m!\033[0m] Applying timezone offset %d (--utc-offset)" % args.utc_offset)
jsono['user_utc_offset_set'] = "Applying timezone offset %d (--utc-offset)" % args.utc_offset
cprint("[+] statuses_count : \033[1m%s\033[0m" % user_info.statuses_count)
jsono['status_count'] = user_info.statuses_count
# Will retreive all Tweets from account (or max limit)
num_tweets = numpy.amin([args.limit, user_info.statuses_count])
cprint("[+] Retrieving last %d tweets..." % num_tweets)
jsono['status_retrieving'] = num_tweets
# Download tweets
get_tweets(twitter_api, args.name, save_file, limit=num_tweets)
cprint("[+] Downloaded %d tweets from %s to %s (%d days)" % (num_tweets, start_date, end_date, (end_date - start_date).days))
jsono['status_start_date'] = "%s" % start_date
jsono['status_end_date'] = "%s" % end_date
jsono['status_days'] = "%s" % (end_date - start_date).days
# Checking if we have enough data (considering it's good to have at least 30 days of data)
if (end_date - start_date).days < 30 and (num_tweets < user_info.statuses_count):
cprint("[\033[91m!\033[0m] Looks like we do not have enough tweets from user, you should consider retrying (--limit)")
jsono['status_note'] = "Looks like we do not have enough tweets from user, you should consider retrying (--limit)"
if (end_date - start_date).days != 0:
cprint("[+] Average number of tweets per day: \033[1m%.1f\033[0m" % (num_tweets / float((end_date - start_date).days)))
jsono['status_average_tweets_per_day'] = (num_tweets / float((end_date - start_date).days))
# Print activity distrubution charts
if args.json is False:
export_string("")
print_charts(activity_hourly, "Daily activity distribution (per hour)")
print_charts(activity_weekly, "Weekly activity distribution (per day)", weekday=True)
jsono["activity_hourly"] = activity_hourly
jsono["activity_weekly"] = activity_weekly
cprint("[+] Detected languages (top 5)")
print_stats(detected_langs)
jsono["top_languages"] = detected_langs
cprint("[+] Detected sources (top 10)")
print_stats(detected_sources, top=10)
jsono["top_sources"] = detected_sources
cprint("[+] There are \033[1m%d\033[0m geo enabled tweet(s)" % geo_enabled_tweets)
jsono['geo_enabled_tweet_count'] = geo_enabled_tweets
if len(detected_places) != 0:
cprint("[+] Detected places (top 10)")
print_stats(detected_places, top=10)
jsono["top_places"] = detected_places
cprint("[+] Top 10 hashtags")
print_stats(detected_hashtags, top=10)
jsono["top_hashtags"] = detected_hashtags
if not args.no_retweets:
cprint("[+] @%s did \033[1m%d\033[0m RTs out of %d tweets (%.1f%%)" % (args.name, retweets, num_tweets, (float(retweets) * 100 / num_tweets)))
jsono['rt_count'] = retweets
# Converting users id to screen_names
retweeted_users_names = {}
for k in retweeted_users.keys():
retweeted_users_names[id_screen_names[k]] = retweeted_users[k]
cprint("[+] Top 5 most retweeted users")
print_stats(retweeted_users_names, top=5)
jsono["top_retweeted_users"] = retweeted_users_names
mentioned_users_names = {}
for k in mentioned_users.keys():
mentioned_users_names[id_screen_names[k]] = mentioned_users[k]
cprint("[+] Top 5 most mentioned users")
print_stats(mentioned_users_names, top=5)
jsono["top_mentioned_users"] = mentioned_users_names
cprint("[+] Most referenced domains (from URLs)")
print_stats(detected_domains, top=6)
jsono["top_referenced_domains"] = detected_domains
if args.friends:
max_friends = numpy.amin([user_info.friends_count, 300])
cprint("[+] Getting %d @%s's friends data..." % (max_friends, args.name))
try:
get_friends(twitter_api, args.name, limit=max_friends)
except tweepy.error.TweepError as e:
if e[0][0]['code'] == 88:
cprint("[\033[91m!\033[0m] Rate limit exceeded to get friends data, you should retry in 15 minutes")
jsono['friend_rate_note'] = "Rate limit exceeded to get friends data, you should retry in 15 minutes"
raise
cprint("[+] Friends languages")
print_stats(friends_lang, top=6)
jsono["top_friends_languages"] = friends_lang
cprint("[+] Friends timezones")
print_stats(friends_timezone, top=8)
jsono["top_friend_timezones"] = friends_timezone
if args.json is not False:
print(json.dumps(jsono))
export_write()
if args.save:
save_file.seek(-1, os.SEEK_END) # drop last ,
save_file.truncate()
save_file.write("]")
save_file.close()
if __name__ == '__main__':
try:
main()
except tweepy.error.TweepError as e:
cprint("[\033[91m!\033[0m] Twitter error: %s" % e)
except Exception as e:
cprint("[\033[91m!\033[0m] Error: %s" % e)