news-analysis.py

from typing import Dict

# import for environment variables and waiting
from time import sleep
from os.path import join, dirname, isfile

# used to parse XML feeds
import xml.etree.ElementTree as ET

# we use it to make async http requests
import aiohttp

# allows us to make our functions async
import asyncio

# date modules that we'll most likely need
from datetime import datetime

# used to grab the XML url list from a CSV file
import csv

# used to save and load coins_in_hand dictionary
import json

# numpy for sums and means
import numpy as np

# nlp library to analyse sentiment
import pytz
from nltk.sentiment import SentimentIntensityAnalyzer

# needed for the binance API
from binance.client import Client
from binance.enums import *

# used for binance websocket
from binance.websockets import BinanceSocketManager

# used for executing the code
from itertools import count

# we use it to time our parser execution speed
from timeit import default_timer as timer

from urllib.request import Request, urlopen

import ssl
ssl._create_default_https_context = ssl._create_unverified_context

# Load Cipher to keep keys from existing on OS in plain text.
from utilities.cipher import initialize_lock_and_key_ciphers, Cipher, load_resource
CIPHERS: Dict[str, Cipher] = initialize_lock_and_key_ciphers()

# Use testnet (change to True) or live (change to False)?
TEST_NET: bool = True

# get binance key and secret from environment variables for testnet and live
RESOURCE_DIRECTORY: str = join(dirname(__file__), "resources")
API_KEYS: Dict[str, str] = load_resource(file_name_cipher=CIPHERS['file_name'],
                                         data_cipher=CIPHERS['data'],
                                         resource_file_name="binance.yaml")
CRYPTO_FEEDS_RESOURCE_FILE_NAME: str = join(RESOURCE_DIRECTORY, "Crypto feeds.csv")

# Authenticate with the client
client = Client(API_KEYS["API Key"], API_KEYS["Secret Key"])

############################################
#     USER INPUT VARIABLES LIVE BELOW      #
# You may edit those to configure your bot #
############################################


# select what coins to look for as keywords in articles headlines
# The key of each dict MUST be the symbol used for that coin on Binance
# keywords are case sensitive
keywords = {'DOT': ['DOT', "Polkadot", "polkadot"],
            'BTC': ['BTC', 'bitcoin', 'Bitcoin', 'BITCOIN'],
            'ONE': ['Harmony', 'ONE'],
            'ETH': ['ETH', 'Ethereum', "etereum", "Vitalik Buterin", "ETH2.0", "ETH2"],
            'ADA': ["Cardano", "ADA", "cardano", "ada", "Charles Hoskinson"],
            "ALGO": ["Algorand", "ALGO", "algo", "algorand"],
            "ATOM": ["Cosmos", "ATOM", "atom", "COSMOS", "cosmos"],
            'BNB' : ['BNB', 'Binance Coin', "binance coin"],
            'SOL': ['SOL', 'Solana', "solana"],
            'MATIC': ['Polygon', 'MATIC', 'polygon', "Layer 2"],
            "LINK": ["ChainLink", "chain link", "chainlink", "LINK"]}

# The Buy amount in the PAIRING symbol, by default USDT
# 100 will for example buy the equivalent of 100 USDT in Bitcoin.
QUANTITY = 100

# define what to pair each coin to
# AVOID PAIRING WITH ONE OF THE COINS USED IN KEYWORDS
PAIRING = 'USDT'

# define how positive the news should be in order to place a trade
# the number is a compound of neg, neu and pos values from the nltk analysis
# input a number between -1 and 1
SENTIMENT_THRESHOLD = 0.1
NEGATIVE_SENTIMENT_THRESHOLD = -0.1

# define the minimum number of articles that need to be analysed in order
# for the sentiment analysis to qualify for a trade signal
# avoid using 1 as that's not representative of the overall sentiment
MINUMUM_ARTICLES = 1

# define how often to run the code (check for new + try to place trades)
# in minutes
REPEAT_EVERY = 1.

# define how old an article can be to be included
# in hours
HOURS_PAST = 24


# coins that bought by the bot since its start
coins_in_hand  = {}

# path to the saved coins_in_hand file
coins_in_hand_file_path = join(RESOURCE_DIRECTORY, 'coins_in_hand.json')

# if saved coins_in_hand json file exists then load it
if isfile(coins_in_hand_file_path):
    with open(coins_in_hand_file_path) as file:
        coins_in_hand = json.load(file)

# and add coins from actual keywords if they aren't in coins_in_hand dictionary already
for coin in keywords:
    if coin not in coins_in_hand:
        coins_in_hand[coin] = 0

# current price of CRYPTO pulled through the websocket
CURRENT_PRICE = {}


def ticker_socket(msg):
    '''Open a stream for financial information for CRYPTO'''
    if msg['e'] != 'error':
        global CURRENT_PRICE
        CURRENT_PRICE['{0}'.format(msg['s'])] = msg['c']
    else:
        print('error')


# connect to the websocket client and start the socket
bsm = BinanceSocketManager(client)
for coin in keywords:
    conn_key = bsm.start_symbol_ticker_socket(coin+PAIRING, ticker_socket)
bsm.start()


'''For the amount of CRYPTO to trade in USDT'''
lot_size = {}

'''Find step size for each coin
For example, BTC supports a volume accuracy of
0.000001, while XRP only 0.1
'''
for coin in keywords:

    try:
        info = client.get_symbol_info(coin+PAIRING)
        step_size = info['filters'][2]['stepSize']
        lot_size[coin+PAIRING] = step_size.index('1') - 1

        if lot_size[coin+PAIRING]<0:
            lot_size[coin+PAIRING]=0

    except:
        pass
for coin in keywords:
    try:
        info = client.get_symbol_info(coin)
        step_size = info['filters'][2]['stepSize']
        lot_size[coin] = step_size.index('1') - 1

        if lot_size[coin]<0:
            lot_size[coin]=0

    except:
        pass



def calculate_one_volume_from_lot_size(coin, amount):
    if coin not in lot_size:
        return float('{:.1f}'.format(amount))

    else:
        return float('{:.{}f}'.format(amount, lot_size[coin]))


def calculate_volume():
    #while CURRENT_PRICE == {}:
    #    print('Connecting to the socket...')
    #    time.sleep(3)

    #else:
        volume = {}
        for coin in CURRENT_PRICE:
            volume[coin] = float(QUANTITY / float(CURRENT_PRICE[coin]))
            volume[coin] = calculate_one_volume_from_lot_size(coin, volume[coin])

        return volume


# load the csv file containg top 100 crypto feeds
# want to scan other websites?
# Simply add the RSS Feed url to the Crypto feeds.csv file
with open(CRYPTO_FEEDS_RESOURCE_FILE_NAME) as csv_file:
    # open the file
    csv_reader = csv.reader(csv_file)

    # remove any headers
    next(csv_reader, None)

    # create empty list
    feeds = []

    # add each row cotaining RSS url to feeds list
    for row in csv_reader:
        feeds.append(row[0])


# Make headlines global variable as it should be the same across all functions
headlines = {'source': [], 'title': [], 'pubDate' : [] }


async def get_feed_data(session, feed, headers):
    '''
    Get relevent data from rss feed, in async fashion
    :param feed: The name of the feed we want to fetch
    :param headers: The header we want on the request
    :param timeout: The default timout before we give up and move on
    :return: None, we don't need to return anything we append it all on the headlines dict
    '''
    try:
        async with session.get(feed, headers=headers, timeout=60) as response:
            # define the root for our parsing
            #text = await response.text()
            req: Request = Request(feed, headers=headers)
            text: str = urlopen(req).read()
            root = ET.fromstring(text)

            channel = root.find('channel/item/title').text
            pubDate = root.find('channel/item/pubDate').text
            # some jank to ensure no alien characters are being passed
            title = channel.encode('UTF-8').decode('UTF-8')

            # convert pubDat to datetime
            published = datetime.strptime(pubDate.replace("GMT", "+0000"), '%a, %d %b %Y %H:%M:%S %z')
            # calculate timedelta
            time_between = datetime.now(pytz.utc) - published

            #print(f'Czas: {time_between.total_seconds() / (60 * 60)}')

            if time_between.total_seconds() / (60 * 60) <= HOURS_PAST:
                # append the source
                headlines['source'].append(feed)
                # append the publication date
                headlines['pubDate'].append(pubDate)
                # append the title
                headlines['title'].append(title)
                print(channel)

    except Exception as e:
        # Catch any error and also print it
        print(f'Could not parse {feed} error is: {e}')


async def get_headlines():
    '''
    Creates a an async task for each of our feeds which are appended to headlines
    :return: None
    '''
    # add headers to the request for ElementTree. Parsing issues occur without headers
    headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0'
    }

    # A nifty timer to see how long it takes to parse all the feeds
    start = timer()
    async with aiohttp.ClientSession() as session:
        tasks = []
        for feed in feeds:
            task = asyncio.ensure_future(get_feed_data(session, feed, headers))
            tasks.append(task)

        # This makes sure we finish all tasks/requests before we continue executing our code
        await asyncio.gather(*tasks)
    end = timer()
    print("Time it took to parse feeds: ", end - start)


def categorise_headlines():
    '''arrange all headlines scaped in a dictionary matching the coin's name'''
    # get the headlines
    asyncio.run(get_headlines())
    categorised_headlines = {}

    # this loop will create a dictionary for each keyword defined
    for keyword in keywords:
        categorised_headlines['{0}'.format(keyword)] = []

    # keyword needs to be a loop in order to be able to append headline to the correct dictionary
    for keyword in keywords:

        # looping through each headline is required as well
        for headline in headlines['title']:
            # appends the headline containing the keyword to the correct dictionary
            if any(key in headline for key in keywords[keyword]):
                categorised_headlines[keyword].append(headline)

    return categorised_headlines


def analyse_headlines():
    '''Analyse categorised headlines and return NLP scores'''
    sia = SentimentIntensityAnalyzer()
    categorised_headlines = categorise_headlines()

    sentiment = {}

    for coin in categorised_headlines:
        if len(categorised_headlines[coin]) > 0:
            # create dict for each coin
            sentiment['{0}'.format(coin)] = []
            # append sentiment to dict
            for title in categorised_headlines[coin]:
                sentiment[coin].append(sia.polarity_scores(title))

    return sentiment


def compile_sentiment():
    '''Arranges every compound value into a list for each coin'''
    sentiment = analyse_headlines()
    compiled_sentiment = {}

    for coin in sentiment:
        compiled_sentiment[coin] = []

        for item in sentiment[coin]:
            # append each compound value to each coin's dict
            compiled_sentiment[coin].append(sentiment[coin][sentiment[coin].index(item)]['compound'])

    return compiled_sentiment


def compound_average():
    '''Calculates and returns the average compoud sentiment for each coin'''
    compiled_sentiment = compile_sentiment()
    headlines_analysed = {}

    for coin in compiled_sentiment:
        headlines_analysed[coin] = len(compiled_sentiment[coin])

        # calculate the average using numpy if there is more than 1 element in list
        compiled_sentiment[coin] = np.array(compiled_sentiment[coin])

        # get the mean
        compiled_sentiment[coin] = np.mean(compiled_sentiment[coin])

        # convert to scalar
        compiled_sentiment[coin] = compiled_sentiment[coin].item()

    return compiled_sentiment, headlines_analysed




def buy(compiled_sentiment, headlines_analysed):
    '''Check if the sentiment is positive and keyword is found for each handle'''
    volume = calculate_volume()
    for coin in compiled_sentiment:
        # check if the sentiment and number of articles are over the given threshold
        if compiled_sentiment[coin] > SENTIMENT_THRESHOLD and headlines_analysed[coin] >= MINUMUM_ARTICLES and coins_in_hand[coin]==0:
            # check the volume looks correct
            print(f'preparing to buy {volume[coin+PAIRING]} {coin} with {PAIRING} at {CURRENT_PRICE[coin+PAIRING]} for ${float(CURRENT_PRICE[coin+PAIRING])*float(volume[coin+PAIRING])}')

            # try to create a real order if the test orders did not raise an exception
            try:
                if TEST_NET:
                    print(f"Placing {'MARKET'} {'BUY'} of {volume[coin+PAIRING]} {coin+PAIRING}")
                else:
                    buy_limit = client.create_order(
                        symbol=coin+PAIRING,
                        side='BUY',
                        type='MARKET',
                        quantity=volume[coin+PAIRING]
                    )

            #error handling here in case position cannot be placed
            except Exception as e:
                print(e)

            # run the else block if the position has been placed and return some info
            else:
                # adds coin to our portfolio
                coins_in_hand[coin] += volume[coin+PAIRING]

                if not TEST_NET:
                    # retrieve the last order
                    order = client.get_all_orders(symbol=coin+PAIRING, limit=1)

                    if order:
                        # convert order timsestamp into UTC format
                        time = order[0]['time'] / 1000
                        utc_time = datetime.fromtimestamp(time)

                        # grab the price of CRYPTO the order was placed at for reporting
                        bought_at = CURRENT_PRICE[coin+PAIRING]

                        # print order condirmation to the console
                        print(f"order {order[0]['orderId']} has been placed on {coin} with {order[0]['origQty']} at {utc_time} and bought at {bought_at}")
                    else:
                        print('Could not get last order from Binance!')

        else:
            print(f'Sentiment not positive enough for {coin}, or not enough headlines analysed or already bought: {compiled_sentiment[coin]}, {headlines_analysed[coin]}')




def sell(compiled_sentiment, headlines_analysed):
    '''Check if the sentiment is negative and keyword is found for each handle'''
    for coin in compiled_sentiment:

        # check if the sentiment and number of articles are over the given threshold
        if compiled_sentiment[coin] < NEGATIVE_SENTIMENT_THRESHOLD and headlines_analysed[coin] >= MINUMUM_ARTICLES and coins_in_hand[coin]>0:

            # check the volume looks correct
            print(f'preparing to sell {coins_in_hand[coin]} {coin} at {CURRENT_PRICE[coin+PAIRING]} for ${coins_in_hand[coin]*CURRENT_PRICE[coin+PAIRING]} total.')

            amount_to_sell = calculate_one_volume_from_lot_size(coin+PAIRING, coins_in_hand[coin]*99.5/100)

            if TEST_NET:
                # create test order before pushing an actual order
                print(f"Placing {'MARKET'} {'SELL'} of {amount_to_sell} {coin+PAIRING}")
            else:
                test_order = client.create_test_order(symbol=coin+PAIRING, side='SELL', type='MARKET', quantity=amount_to_sell)

            # try to create a real order if the test orders did not raise an exception
            try:
                buy_limit = client.create_order(
                    symbol=coin+PAIRING,
                    side='SELL',
                    type='MARKET',
                    quantity=amount_to_sell
                )

            #error handling here in case position cannot be placed
            except Exception as e:
                print(e)

            # run the else block if the position has been placed and return some info
            else:
                # retrieve the last order

                if TEST_NET:
                    utc_time = datetime.now()

                    sold_at = CURRENT_PRICE[coin+PAIRING]

                    print(f"order {0.} has been placed on {coin} with {coins_in_hand[coin]} coins sold for {sold_at} each at {utc_time}")
                else:
                    order = client.get_all_orders(symbol=coin+PAIRING, limit=1)

                    if order:
                        # convert order timsestamp into UTC format
                        utc_time = datetime.fromtimestamp(order[0]['time'] / 1000.)

                        # grab the price of CRYPTO the order was placed at for reporting
                        sold_at = CURRENT_PRICE[coin+PAIRING]

                        # print order condirmation to the console
                        print(f"order {order[0]['orderId']} has been placed on {coin} with {order[0]['origQty']} coins sold for {sold_at} each at {utc_time}")
                    else:
                        print('Could not get last order from Binance!')
                # set coin to 0
                coins_in_hand[coin]=0


        else:
            print(f'Sentiment not negative enough for {coin}, not enough headlines analysed or not enough {coin} to sell: {compiled_sentiment[coin]}, {headlines_analysed[coin]}')


def save_coins_in_hand_to_file():
    # abort saving if dictionary is empty
    if not coins_in_hand:
        return

    # save coins_in_hand to file
    with open(coins_in_hand_file_path, 'w') as file:
        json.dump(coins_in_hand, file, indent=4)



if __name__ == '__main__':
    print('Press Ctrl-Q to stop the script')
    for i in count():
        compiled_sentiment, headlines_analysed = compound_average()
        print("\nBUY CHECKS:")
        buy(compiled_sentiment, headlines_analysed)
        print("\nSELL CHECKS:")
        sell(compiled_sentiment, headlines_analysed)
        print('\nCurrent bot holdings: ')
        for coin in coins_in_hand:
            if coins_in_hand[coin] > 0:
                print(f'{coin}: {coins_in_hand[coin]}')
        save_coins_in_hand_to_file()
        print(f'\nIteration {i}')
        sleep(60 * REPEAT_EVERY)