# Ubuntu Linux
sudo apt install -y python3 python3-pip
 
# Mac - open a terminal!

# Windows
py main.py
 
# Mac/Linux
python3 main.py

def add(a, b):
	print("{} + {} = {}".format(a, b, a + b))
 
print("Hello World!")
 
add(2, 6)

import nltk
 
nltk.download()

# https://www.nltk.org
 
# Windows
py -m pip install nltk
 
# Mac/Linux
pip install nltk
pip3 install nltk

from nltk.tokenize import sent_tokenize, word_tokenize
 
txt = "Hello Mr. Smith, how are you doing today? The weather is great, and NLTK is awesome. The sky is kinda blue. You shouldn't eat cardboard."
 
sentences = sent_tokenize(txt)
 
print(sentences)
print()
 
for s in sentences:
	print(s)
	words = word_tokenize(s)
	print(words)

from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
 
sentence = "Hello Mr. Smith, how are you doing today?"
sentence = word_tokenize(sentence)
 
stop_words = set(stopwords.words("english"))
print(stop_words)
 
filtered = [w for w in sentence if not w in stop_words]
# Equivalent to:
# filtered = []
# for w in sentence:
# 	if w not in stop_words:
#   	filtered.append(w)
 
print(filtered)

from nltk.stem import PorterStemmer, LancasterStemmer
from nltk.tokenize import word_tokenize
 
ps = PorterStemmer()
ls = LancasterStemmer()
 
words = ["walk", "walker", "walking", "walked", "walkly"]
 
for w in words: print(w, ps.stem(w))
# for w in words: print(w, ls.stem(w))

import nltk
from nltk.corpus import state_union
from nltk.tokenize import PunktSentenceTokenizer
 
train_text = state_union.raw("2005-GWBush.txt")
sample_text = state_union.raw("2006-GWBush.txt")
 
custom_sent_tokenizer = PunktSentenceTokenizer(train_text)
 
tokenized = custom_sent_tokenizer.tokenize(sample_text)
 
def process_content():
    try:
        for i in tokenized:
            words = nltk.word_tokenize(i)
            tagged = nltk.pos_tag(words)
            print(tagged)
    except Exception as e:
        print(str(e))
 
process_content()

import nltk
from nltk.corpus import state_union
from nltk.tokenize import PunktSentenceTokenizer
 
train_text = state_union.raw("2005-GWBush.txt")
sample_text = state_union.raw("2006-GWBush.txt")
 
custom_sent_tokenizer = PunktSentenceTokenizer(train_text)
 
tokenised = custom_sent_tokenizer.tokenize(sample_text)
 
try:
  for i in tokenised:
    words =  nltk.word_tokenize(i)
    tagged = nltk.pos_tag(words)
 
    chunkGram = r"""Chunk: {<RB.?>*<VB.?>*<NNP>+<NN>?}""" # 0+ Any adverb, 0+ Verb, Proper Noun
    chunkParser = nltk.RegexpParser(chunkGram)
    chunked = chunkParser.parse(tagged)
    print(chunked)
 
    chunked.draw()
except Exception as e:
    print(str(e))

import nltk
from nltk.corpus import state_union
from nltk.tokenize import PunktSentenceTokenizer
 
train_text = state_union.raw("2005-GWBush.txt")
sample_text = state_union.raw("2006-GWBush.txt")
 
custom_sent_tokenizer = PunktSentenceTokenizer(train_text)
 
tokenised = custom_sent_tokenizer.tokenize(sample_text)
 
try:
  for i in tokenised[5:]:
    words =  nltk.word_tokenize(i)
    tagged = nltk.pos_tag(words)
 
    chunkGram = r"""Chunk: {<.*>+}
				}<VB.?|IN|DT|TO>+{""" # Exclude these
 
    chunkParser = nltk.RegexpParser(chunkGram)
    chunked = chunkParser.parse(tagged)
 
    chunked.draw()
except Exception as e:
  print(str(e))

import nltk
from nltk.corpus import state_union
from nltk.tokenize import PunktSentenceTokenizer
 
train_text = state_union.raw("2005-GWBush.txt")
sample_text = state_union.raw("2006-GWBush.txt")
 
custom_sent_tokenizer = PunktSentenceTokenizer(train_text)
 
tokenized = custom_sent_tokenizer.tokenize(sample_text)
 
def process_content():
    try:
        for i in tokenized[5:]:
            words =  nltk.word_tokenize(i)
            tagged = nltk.pos_tag(words)
 
            # namedEnt = nltk.ne_chunk(tagged)
            namedEnt = nltk.ne_chunk(tagged, binary=True) # Treat all as named entity
            namedEnt.draw()
    except Exception as e:
        print(str(e))
 
process_content()

from nltk.corpus import gutenberg
from nltk.tokenize import sent_tokenize
 
text = gutenberg.raw("bible-kjv.txt")
tokens = sent_tokenize(text)
 
print(tokens[5:15])
 
# Check out the corpus! - nltk.download()

from nltk.corpus import wordnet
 
syns = wordnet.synsets("program")
 
# Synset
print(syns[0].name())
 
# Word Only
print(syns[0].lemmas()[0].name())
 
# Definition
print(syns[0].definition())
 
# Examples
print(syns[0].examples())
 
 
# Synonyms & Antonyms
synonyms = []
antonyms = []
 
for syn in wordnet.synsets("good"):
    for l in syn.lemmas():
        # print("l: ", l)
        synonyms.append(l.name())
        if (l.antonyms()):
            antonyms.append(l.antonyms()[0].name())
 
print(set(synonyms))
print(set(antonyms))
 
 
# Similarity
w1 = wordnet.synset("ship.n.01")
w2 = wordnet.synset("boat.n.01")
 
print(w1.wup_similarity(w2))
 
w1 = wordnet.synset("ship.n.01")
w2 = wordnet.synset("car.n.01")
 
print(w1.wup_similarity(w2))
 
w1 = wordnet.synset("ship.n.01")
w2 = wordnet.synset("cat.n.01")
 
print(w1.wup_similarity(w2))
 
w1 = wordnet.synset("ship.n.01")
w2 = wordnet.synset("cactus.n.01")
 
print(w1.wup_similarity(w2))

from nltk.stem import WordNetLemmatizer
 
lemmatizer = WordNetLemmatizer()
 
print(lemmatizer.lemmatize("cats"))
print(lemmatizer.lemmatize("cacti"))
print(lemmatizer.lemmatize("geese"))
print(lemmatizer.lemmatize("rocks"))
print(lemmatizer.lemmatize("python"))
print()
print(lemmatizer.lemmatize("better")) # Default pos is NOUN
print(lemmatizer.lemmatize("better", pos="a"))
print(lemmatizer.lemmatize("best", pos="a"))
print(lemmatizer.lemmatize("run"))
print(lemmatizer.lemmatize("run", pos="v"))

import nltk
import random
from nltk.corpus import movie_reviews
 
# One Liner
documents = [(list(movie_reviews.words(fileid)), category)
                for category in movie_reviews.categories()
                for fileid in movie_reviews.fileids(category)]
 
random.shuffle(documents)
 
all_words = []
for w in movie_reviews.words():
    all_words.append(w.lower()) # Convert to lowercase
 
all_words = nltk.FreqDist(all_words) # NLTK Frequency Distribution
 
word_features = list(all_words.keys())[:3000] # Top 3,000 words
 
 
 
def find_features(document):
    words = set(document)
    features = {}
    for w in word_features:
        features[w] = (w in words) # Boolean
 
    return features
 
# print(find_features(movie_reviews.words("neg/cv000_29416.txt")))
 
featuresets = [(find_features(rev), category) for (rev, category) in documents]
 
 
 
training_set = featuresets[:1900] # First 1900
testing_set = featuresets[1900:] # Post 1900
 
# Naive Bayes Algorithm
# Posterior = Prior Occurances x Likelihood / Evidence
 
classifier = nltk.NaiveBayesClassifier.train(training_set)
print("Naive Bayes Accuracy: {:f}%".format((nltk.classify.accuracy(classifier, testing_set))*100))
classifier.show_most_informative_features(15)
# Word  Ratios (Found more in pos or neg)

import nltk
 
# Data Sets
training_set = data[:1900]
testing_set = data[1900:]
 
classifier = nltk.NaiveBayesClassifier.train(training_set)

import nltk
 
# Data Sets
training_set = data[:1900]
testing_set = data[1900:]
 
classifier = nltk.NaiveBayesClassifier.train(training_set)

import nltk
import random
from nltk.corpus import movie_reviews
import pickle
 
from sklearn.naive_bayes import MultinomialNB, GaussianNB, BernoulliNB
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.svm import SVC, LinearSVC, NuSVC
from nltk.classify.scikitlearn import SklearnClassifier
 
# One Liner
documents = [(list(movie_reviews.words(fileid)), category)
                for category in movie_reviews.categories()
                for fileid in movie_reviews.fileids(category)]
 
random.shuffle(documents)
 
all_words = []
for w in movie_reviews.words():
    all_words.append(w.lower()) # Convert to lowercase
 
all_words = nltk.FreqDist(all_words) # NLTK Frequency Distribution
 
word_features = list(all_words.keys())[:3000] # Top 3,000 words
 
 
 
def find_features(document):
    words = set(document)
    features = {}
    for w in word_features:
        features[w] = (w in words) # Boolean
 
    return features
 
# print(find_features(movie_reviews.words("neg/cv000_29416.txt")))
 
featuresets = [(find_features(rev), category) for (rev, category) in documents]
 
 
 
training_set = featuresets[:1900] # First 1900
testing_set = featuresets[1900:] # Post 1900
 
# Naive Bayes Algorithm
# Posterior = Prior Occurances x Likelihood / Evidence
 
classifier = nltk.NaiveBayesClassifier.train(training_set)
 
# Open Pickle Classifier
classifier_f = open("naivebayes.pickle", "rb")
classifier = pickle.load(classifier_f)
classifier_f.close()
 
print("ORIGINAL Naive Bayes Accuracy: {:.2f}%".format((nltk.classify.accuracy(classifier, testing_set))*100))
classifier.show_most_informative_features(15)
# Word  Ratios (Found more in pos or neg)
 
 
 
MNB_classifier = SklearnClassifier(MultinomialNB()) # Wrapper converts to NLTK classifier
MNB_classifier = MNB_classifier.train(training_set)
print("MNB_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(MNB_classifier, testing_set)*100))
 
BernoulliNB_classifier = SklearnClassifier(BernoulliNB())
BernoulliNB_classifier = BernoulliNB_classifier.train(training_set)
print("BernoulliNB_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(BernoulliNB_classifier, testing_set)*100))
 
LogisticRegression_classifier = SklearnClassifier(LogisticRegression(solver="liblinear"))
LogisticRegression_classifier = LogisticRegression_classifier.train(training_set)
print("LogisticRegression_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(LogisticRegression_classifier, testing_set)*100))
 
SGDClassifier_classifier = SklearnClassifier(SGDClassifier())
SGDClassifier_classifier = SGDClassifier_classifier.train(training_set)
print("SGDClassifier_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(SGDClassifier_classifier, testing_set)*100))
 
# Very Inaccurate (< 50%)
SVC_classifier = SklearnClassifier(SVC(gamma="auto"))
SVC_classifier = SVC_classifier.train(training_set)
print("SVC_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(SVC_classifier, testing_set)*100))
 
LinearSVC_classifier = SklearnClassifier(LinearSVC())
LinearSVC_classifier = LinearSVC_classifier.train(training_set)
print("LinearSVC_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(LinearSVC_classifier, testing_set)*100))
 
NuSVC_classifier = SklearnClassifier(NuSVC(gamma="auto"))
NuSVC_classifier = NuSVC_classifier.train(training_set)
print("NuSVC_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(NuSVC_classifier, testing_set)*100))

import nltk
import random
from nltk.corpus import movie_reviews
import pickle
 
from sklearn.naive_bayes import MultinomialNB, GaussianNB, BernoulliNB
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.svm import SVC, LinearSVC, NuSVC
from nltk.classify.scikitlearn import SklearnClassifier
 
from nltk.classify import ClassifierI
from statistics import mode
 
class VoteClassifier(ClassifierI):
    def __init__(self, *classifiers):
        self._classifiers = classifiers
 
    def classify(self, features):
        votes = []
        for c in self._classifiers:
            v = c.classify(features)
            votes.append(v)
        return mode(votes)
    def confidence(self, features):
        votes = []
        for c in self._classifiers:
            v = c.classify(features)
            votes.append(v)
        choice_votes = votes.count(mode(votes))
        conf = choice_votes / len(votes)
        return conf
 
# One Liner
documents = [(list(movie_reviews.words(fileid)), category)
                for category in movie_reviews.categories()
                for fileid in movie_reviews.fileids(category)]
 
random.shuffle(documents)
 
all_words = []
for w in movie_reviews.words():
    all_words.append(w.lower()) # Convert to lowercase
 
all_words = nltk.FreqDist(all_words) # NLTK Frequency Distribution
 
word_features = list(all_words.keys())[:3000] # Top 3,000 words
 
 
 
def find_features(document):
    words = set(document)
    features = {}
    for w in word_features:
        features[w] = (w in words) # Boolean
 
    return features
 
# print(find_features(movie_reviews.words("neg/cv000_29416.txt")))
 
featuresets = [(find_features(rev), category) for (rev, category) in documents]
 
 
 
training_set = featuresets[:1900] # First 1900
testing_set = featuresets[1900:] # Post 1900
 
# Naive Bayes Algorithm
# Posterior = Prior Occurances x Likelihood / Evidence
 
classifier = nltk.NaiveBayesClassifier.train(training_set)
 
# Open Pickle Classifier
classifier_f = open("naivebayes.pickle", "rb")
classifier = pickle.load(classifier_f)
classifier_f.close()
 
print("ORIGINAL Naive Bayes Accuracy: {:.2f}%".format((nltk.classify.accuracy(classifier, testing_set))*100))
classifier.show_most_informative_features(15)
# Word  Ratios (Found more in pos or neg)
 
 
 
MNB_classifier = SklearnClassifier(MultinomialNB()) # Wrapper converts to NLTK classifier
MNB_classifier = MNB_classifier.train(training_set)
print("MNB_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(MNB_classifier, testing_set)*100))
 
BernoulliNB_classifier = SklearnClassifier(BernoulliNB())
BernoulliNB_classifier = BernoulliNB_classifier.train(training_set)
print("BernoulliNB_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(BernoulliNB_classifier, testing_set)*100))
 
LogisticRegression_classifier = SklearnClassifier(LogisticRegression(solver="liblinear"))
LogisticRegression_classifier = LogisticRegression_classifier.train(training_set)
print("LogisticRegression_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(LogisticRegression_classifier, testing_set)*100))
 
# Stoicastic Gradient Descent
SGDClassifier_classifier = SklearnClassifier(SGDClassifier())
SGDClassifier_classifier = SGDClassifier_classifier.train(training_set)
print("SGDClassifier_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(SGDClassifier_classifier, testing_set)*100))
 
# Very Inaccurate (< 50%)
# SVC_classifier = SklearnClassifier(SVC(gamma="auto"))
# SVC_classifier = SVC_classifier.train(training_set)
# print("SVC_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(SVC_classifier, testing_set)*100))
 
LinearSVC_classifier = SklearnClassifier(LinearSVC())
LinearSVC_classifier = LinearSVC_classifier.train(training_set)
print("LinearSVC_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(LinearSVC_classifier, testing_set)*100))
 
NuSVC_classifier = SklearnClassifier(NuSVC(gamma="auto"))
NuSVC_classifier = NuSVC_classifier.train(training_set)
print("NuSVC_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(NuSVC_classifier, testing_set)*100))
 
 
 
voted_classifier = VoteClassifier(classifier, MNB_classifier, BernoulliNB_classifier, LogisticRegression_classifier, SGDClassifier_classifier, LinearSVC_classifier, NuSVC_classifier)
print("voted_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(voted_classifier, testing_set)*100))
 
print("testing_set[0][0] Classification:", voted_classifier.classify(testing_set[0][0]), "Confidence: {:.5f}%".format(voted_classifier.confidence(testing_set[0][0])*100))
print("testing_set[1][0] Classification:", voted_classifier.classify(testing_set[1][0]), "Confidence: {:.5f}%".format(voted_classifier.confidence(testing_set[1][0])*100))
print("testing_set[2][0] Classification:", voted_classifier.classify(testing_set[2][0]), "Confidence: {:.5f}%".format(voted_classifier.confidence(testing_set[2][0])*100))
print("testing_set[3][0] Classification:", voted_classifier.classify(testing_set[3][0]), "Confidence: {:.5f}%".format(voted_classifier.confidence(testing_set[3][0])*100))
print("testing_set[4][0] Classification:", voted_classifier.classify(testing_set[4][0]), "Confidence: {:.5f}%".format(voted_classifier.confidence(testing_set[4][0])*100))
print("testing_set[5][0] Classification:", voted_classifier.classify(testing_set[5][0]), "Confidence: {:.5f}%".format(voted_classifier.confidence(testing_set[5][0])*100))

#File: sentiment_mod.py
 
import nltk
import random
# from nltk.corpus import movie_reviews
import pickle
from sklearn.naive_bayes import MultinomialNB, GaussianNB, BernoulliNB
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.svm import SVC, LinearSVC, NuSVC
from nltk.classify.scikitlearn import SklearnClassifier
from nltk.classify import ClassifierI
from statistics import mode
from nltk.tokenize import word_tokenize
 
def load_pickle(filename):
    classifier_f = open("pickle/"+filename, "rb")
    classifier = pickle.load(classifier_f)
    classifier_f.close()  
    return classifier
 
def save_pickle(classifier, filename):
    save_classifier = open("pickle/"+filename, "wb")
    pickle.dump(classifier, save_classifier)
    save_classifier.close()
 
 
class VoteClassifier(ClassifierI):
    def __init__(self, *classifiers):
        self._classifiers = classifiers
 
    def classify(self, features):
        votes = []
        for c in self._classifiers:
            v = c.classify(features)
            votes.append(v)
        return mode(votes)
    def confidence(self, features):
        votes = []
        for c in self._classifiers:
            v = c.classify(features)
            votes.append(v)
        choice_votes = votes.count(mode(votes))
        conf = choice_votes / len(votes)
        return conf
 
def find_features(document):
    words = word_tokenize(document)
    features = {}
    for w in word_features:
        features[w] = (w in words) # Boolean
 
    return features
 
documents = load_pickle("documents.pickle")
 
word_features = load_pickle("word_features_5k.pickle")
 
 
# featuresets = [(find_features(rev), category) for (rev, category) in documents]
# save_pickle(featuresets, "featuresets.pickle")
featuresets = load_pickle("featuresets.pickle")
 
random.shuffle(featuresets)
 
training_set = featuresets[:10000]
testing_set = featuresets[10000:]
 
# Naive Bayes Algorithm: Posterior = Prior Occurances x Likelihood / Evidence
 
classifier = load_pickle("short_reviews.pickle")
 
MNB_classifier = load_pickle("MNB_classifier.pickle")
# print("MNB_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(MNB_classifier, testing_set)*100))
 
BernoulliNB_classifier = load_pickle("BernoulliNB_classifier.pickle")
# print("BernoulliNB_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(BernoulliNB_classifier, testing_set)*100))
 
LogisticRegression_classifier = load_pickle("LogisticRegression_classifier.pickle")
# print("LogisticRegression_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(LogisticRegression_classifier, testing_set)*100))
 
LinearSVC_classifier = load_pickle("LinearSVC_classifier.pickle")
# print("LinearSVC_classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(LinearSVC_classifier, testing_set)*100))
 
 
voted_classifier = VoteClassifier(
    classifier,
    MNB_classifier,
    BernoulliNB_classifier,
    LogisticRegression_classifier,
    LinearSVC_classifier,
)
print("Vote Classifier Accuracy: {:.2f}%".format(nltk.classify.accuracy(voted_classifier, testing_set)*100))
 
def sentiment(text):
    feats = find_features(text)
    return voted_classifier.classify(feats),voted_classifier.confidence(feats)
 

import sentiment_mod as s
 
# Positive
print(s.sentiment("This movie was awesome! The acting was great, plot was wonderful, and there were pythons...so yea!"))
 
# Negative
print(s.sentiment("This movie was utter junk. There were absolutely 0 pythons. I don't see what the point was at all. Horrible movie, 0/10"))

from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener
import json
 
import sentiment_mod as s
 
#consumer key, consumer secret, access token, access secret
#ckey, csecret, akey, asecret
from secret import *
 
class listener(StreamListener):
    def on_data(self, data):
        all_data = json.loads(data)
 
        tweet = all_data["text"]
        sentiment_value, confidence = s.sentiment(tweet)
        print(tweet, sentiment_value, confidence)
 
        if confidence*100 >= 95:
            output = open("twitter-out.txt", "a")
            output.write(sentiment_value)
            output.write('\n')
            output.close()
 
        return True
 
    def on_error(self, status):
        print(status)
 
auth = OAuthHandler(ckey, csecret)
auth.set_access_token(atoken, asecret)
 
open("twitter-out.txt", "w").close() # Clear file
 
twitterStream = Stream(auth, listener())
 
word = input("Keyword(s) to track: ")
twitterStream.filter(track=[word])

import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib import style
import time
 
style.use("ggplot")
 
fig = plt.figure()
ax1 = fig.add_subplot(1,1,1)
 
fig.canvas.set_window_title("Live Twitter Sentiment")
 
def animate(i):
    pullData = open("twitter-out.txt","r").read()
    lines = pullData.split('\n')
 
    xar = []
    yar = []
 
    x = 0
    y = 0
 
    for l in lines[-200:]:
        x += 1 # Move x along
        if "pos" in l:
            y += 1 
        elif "neg" in l:
            y -= 1
 
        xar.append(x)
        yar.append(y)
        
    ax1.clear()
    ax1.plot(xar,yar)
ani = animation.FuncAnimation(fig, animate, interval=1000)
plt.show()