import praw
import pandas as pd
import datetime as dt 

reddit = praw.Reddit(
    user_agent="Comment Extraction (by u/USERNAME)",
    client_id="*********",
    client_secret="*********",
    username="*********",
    password="*********",
)

url = "https://www.reddit.com/r/movies/comments/o7e258/official_discussion_f9_the_fast_saga_spoilers/"
submission = reddit.submission(url=url)

from praw.models import MoreComments
from datetime import datetime

# Create temporary list to hold the comments data.
list_comments=[]
list_time=[]

# Store in a dictionary.
dict_collect = {'comments':list_comments,
                'time': list_time,
               }

# Loop through each top level comment.
for top_level_comment in submission.comments:
    if isinstance(top_level_comment, MoreComments):
        continue
    # Append comment to list
    list_comments.append(top_level_comment.body)
    
    # Convert and append time to list
    time = (datetime.utcfromtimestamp(top_level_comment.created_utc).strftime('%Y-%m-%d %H:%M:%S'))
    list_time.append(time)

df = pd.DataFrame.from_dict(dict_collect)
df

df.to_csv('dataset/fast_9_review.csv',index = False)

eda = pd.read_csv('dataset/fast_9_review.csv')

eda.head()

eda['word_count'] = eda['comments'].apply(lambda x: len(str(x).split(" ")))

eda.head()

print('Average Word count: '+str(int(eda['word_count'].mean()))+ ' words')

def avg_word(comments):
  words = comments.split()
  return int((sum(len(word) for word in words) / len(words)))

# Calculate average words
eda['avg_word_len'] = eda['comments'].apply(lambda x: avg_word(x))

eda.head()

eda['char_count'] = eda['comments'].str.len()

eda.head()

print('Average character count: '+str(int(eda['char_count'].mean()))+ ' words')

# Import stopwords
from nltk.corpus import stopwords

stop_words = stopwords.words('english')
eda['stopword_count'] = eda['comments'].apply(lambda x: len([x for x in x.split() if x in stop_words]))

eda.head()

eda.describe()

data = pd.read_csv('dataset/fast_9_review.csv')

data.head()

# Converting each comments to lowercase
data['lowercased'] = data['comments'].apply(lambda x: " ".join(x.lower() for x in x.split()))

data.head()

# remove punctuations from comments
data['nopunc'] = data['lowercased'].str.replace('[^\w\s]', '')

data.head()

# Import stopwords
from nltk.corpus import stopwords
stop_words = stopwords.words('english')

data['nopunc_nostop'] = data['nopunc'].apply(lambda x: " ".join(x for x in x.split() if x not in stop_words))

data.head()

# View the top 30 words used
freq= pd.Series(" ".join(data['nopunc_nostop']).split()).value_counts()[:30]
freq

other_stopwords = ['actually', 'time', 'one', 'get', 'got', 'even', 'time']

data['nopunc_nostop_nocommon'] = data['nopunc_nostop'].apply(lambda x: "".join(" ".join(x for x in x.split() if x not in other_stopwords)))

data.head()

# Import textblob
from textblob import Word

# Lemmatize final review format
data['cleaned_comments'] = data['nopunc_nostop_nocommon'].apply(lambda x: " ".join([Word(word).lemmatize() for word in x.split()]))

data.head()

data[['cleaned_comments','time']].to_csv('dataset/fast_9_review_cleaned.csv',index = False)

df = pd.read_csv('dataset/fast_9_review_cleaned.csv')

df

# Calculate polarity
from textblob import TextBlob
df['polarity'] = df['cleaned_comments'].apply(lambda x: TextBlob(x).sentiment[0])

# Calculate subjectivity
df['subjectivity'] = df['cleaned_comments'].apply(lambda x: TextBlob(x).sentiment[1])

df.head()

# Summary statictics of the scores
df.describe()

# Add polarity label

def polar_label(polar):
  if (polar<-0.5):
    return "negative"
  elif (polar>=-0.5) and (polar<0):
    return "weak negative"
  elif (polar==0):
    return "neutral"
  elif (polar>0)and(polar<=0.5):
    return "weak positive"
  else:
    return "positive"

df['polar_label'] = df['polarity'].apply(lambda x: polar_label(x))

# Add subjectivity label

def subj_label(subj):
  if subj<0.5:
    return "objective"
  elif subj==0.5:
    return "neutral"
  else:
    return "subjective"

df['subj_label'] = df['subjectivity'].apply(lambda x: subj_label(x))

df.head()

# count of polar label
df['polar_label'].value_counts()

import seaborn as sns
import matplotlib.pyplot as plt

plt.figure(figsize=(10,5),facecolor='1')
plt.title('Polarity score distribution')

sns.kdeplot(
   data=df, x="polarity", 
   fill=True, palette="Set1",
   alpha=.5, linewidth=1)

plt.show()

plt.figure(figsize=(10,5),facecolor='1')
plt.title('Subjectivity score distribution')

sns.kdeplot(
   data=df, x="subjectivity", 
   fill=True, palette="Set1",
   alpha=.5, linewidth=1)

plt.show()

plt.figure(figsize=(10,5),facecolor='1')
plt.title('Count of polarity score')

sns.countplot(
   data=df, x="polar_label",palette="Set1")

plt.show()

plt.figure(figsize=(10,5),facecolor='1')
plt.title('Count of polarity score')

sns.countplot(
   data=df, x="subj_label",palette="Set1")

plt.show()

from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

# Text of all words in column Tweets

df_neg = df[df['polar_label']=='negative']

text = " ".join(review for review in df_neg.cleaned_comments.astype(str))
print ("There are {} words in the combination of all cells in column Tweets.".format(len(text)))

# Create stopword list:
# remove words that we want to exclude

stopwords = set(STOPWORDS)

# Generate a word cloud image

wordcloud = WordCloud(stopwords=stopwords, background_color="white", width=800, height=400,colormap='Set1').generate(text)

# Display the generated image:
# the matplotlib way:

fig=plt.figure(figsize=(10,5))
plt.tight_layout(pad=0)
plt.axis("off")
plt.imshow(wordcloud, interpolation='bilinear')
plt.show()

# Text of all words in column Tweets

df_neg = df[df['polar_label']=='positive']

text = " ".join(review for review in df_neg.cleaned_comments.astype(str))
print ("There are {} words in the combination of all cells in column Tweets.".format(len(text)))

# Create stopword list:
# remove words that we want to exclude

stopwords = set(STOPWORDS)

# Generate a word cloud image

wordcloud = WordCloud(stopwords=stopwords, background_color="white", width=800, height=400,colormap='plasma').generate(text)

# Display the generated image:
# the matplotlib way:

fig=plt.figure(figsize=(10,5))
plt.tight_layout(pad=0)
plt.axis("off")
plt.imshow(wordcloud, interpolation='bilinear')
plt.show()