import scrapy
from scrapy.linkextractors import LinkExtractor
from bs4 import BeautifulSoup
import requests
import re
from scrapy.crawler import CrawlerProcess

class LyricSpider(scrapy.Spider):
    name='lyrics_spider'

    start_urls = []
    
    def __init__(self, form_inp='', **kwargs): # The category variable will have the input 
    form_inp = re.sub(', ', ',', form_inp)
    artists = form_inp.split(',')

    for artist in artists:
        artist = re.sub(' ', '-', artist).capitalize()
        self.start_urls.append('https://genius.com/artists/' + artist + '/')
    print(self.start_urls)
    super().__init__(**kwargs)

    def parse(self,response):
        '''parse method; begins at our initial page 
        goes to our desired next page (artist's albums)
        '''
        #creating our soup object
        page = requests.get(response.url)
        soup = BeautifulSoup(page.content, "html.parser")
        #gets links to all albums on the artist's page
        next_page = [i['href'] for i in soup.find_all("a", href=lambda href: href and "https://genius.com/albums/" in href)]

        #yields request for each url in list
        for link in next_page:
            yield scrapy.Request(link, callback=self.parse_songs)

    def parse_songs(self, response):
        '''generates and visits sites of all songs in the album
        '''

        #create soup object
        album_page = requests.get(response.url)
        soup2 = BeautifulSoup(album_page.content, "html.parser")
        #gets all links for songs on the page 
        songs = [i['href'] for i in soup2.find_all("a", href=lambda href: href and "https://genius.com/" and "lyrics" in href)]
        #gets artist name as list
        artist = response.css("a.header_with_cover_art-primary_info-primary_artist::text").get()
        
        #get name inside list
        name = artist[0:]
        #convert to lowercase
        name = name.lower()
        #capitalize only first letter
        name = name[0:].capitalize()
        #replace spaces with hyphen
        name= re.sub(" ", "-", name)

        #yields request for each url in list if it's a song by our desired artist(s)
        for new_link in songs:
            if name not in new_link:
                songs.remove(new_link)
            else:
                yield scrapy.Request(new_link, callback=self.parse_song_page)

    def parse_song_page(self, response):
        '''creates dictionary with lyrics in each song and corresponding song name
        '''
        #extracts song name from header
        song_name = response.css('span.SongHeaderVariantdesktop__HiddenMask-sc-12tszai-10.bFjDxc::text').get()
        #extracts lyrics from page
        song_lyrics = response.css('span.ReferentFragmentVariantdesktop__Highlight-sc-1837hky-1.jShaMP::text').getall() 

        yield{

        "song": song_name,
        "lyrics": song_lyrics

        }

import pandas as pd

lyrics = pd.read_csv("lyrics.csv")

lyrics

from itertools import islice
import re
import numpy as np
import tensorflow as tf
import string

pd.set_option("display.max_rows", None, "display.max_columns", None)

def consec_chars(chars, n):
  '''generates n-long tuples with a sliding window'''
  for i in range(len(chars) - (n-1)):
    yield chars[i:i+n]

def ngram_transition_matrix(chars, n):
  '''Generates a transition matrix given a list of characters'''
  df = pd.DataFrame(consec_chars(chars, n), columns = [str(i) for i in range(1,n+1)])
  
  # for a group of n - 1 preceding words, count the frequency of the following word
  ngroup = list(df)[:n-1]
  counts = df.groupby(ngroup)[str(n)].value_counts()

  # compute relative frequency from counts and 
  # produce a transition matrix using unstack
  probs = (counts / counts.groupby(level=ngroup).sum()).unstack()
  return probs.fillna(0)

#Cleaning of text (removes punctuation and makes everything lowercase )

def standardization(text):
    text = text.lower()
    text = re.sub(r'[\(\[].*?[\)\]]', '', text)
    text = re.sub(r',\s', ',', text)
    return text

def next_word(tup, mat):
  '''sample next word given previous n-1 characters'''
  poss = mat.loc[tup,]
  return np.random.choice(list(mat), p = poss)

def generate_block(title, start, length, mat, n):
  '''generates a new text of the given length'''
  start_tup = tuple(start)
  # get current MultiIndex (or regular index if doing bigram)
  curr = start_tup if n > 2 else start
  lyrics = '[' + title + ']' + '\n'
  lyrics += ''.join(start_tup)
  j = 0
  # constructs a sentence for the given seed
  while j < length:
    nxt = next_word(curr, mat)
    lyrics += nxt + ('\n' if nxt == ',' else '')
    # shift window by one to predict next word
    curr = nxt if n == 2 else (*curr[1:], nxt)

    # to ensure the ending of a section does not cut off any words,
    # run the loop until the last character is a space
    if j < length - 1 or curr[-1] == ' ':
      j += 1

  # Corrects spacing at the beginning of lines and in between commas
  lyrics = re.sub(',', ', ', lyrics)
  lyrics = re.sub(' ,', ',', lyrics)
  return re.sub('\n\s', '\n', lyrics) + '\n\n'

def random_start(text, n):
  '''generate the starting word for a song section'''
  spaces = [i for i in range(len(text)) if text[i] == ' ']
  j = np.random.choice(spaces)
  return text[j+1:j+n]

def generate_song(text, n=2, len_verse=500, len_chorus=100, num_verses=2):
  '''simulate the Markov chain and format appropriately'''
  np.random.seed()

  # create random starting phrases
  text = standardization(text)
  starts = [random_start(text, n) for i in range(num_verses+1)] 
  chars = list(text)

  mat = ngram_transition_matrix(chars, n)
  
  song = ""
  chorus = generate_block('Chorus', starts[0], len_chorus, mat, n)

  for i in range(num_verses):
    song += generate_block('Verse ' + str(i+1), starts[i+1], len_verse, mat, n)
    song += chorus

  return song

lyrics_df = pd.read_json(r'./file.json')
lyrics_df['lyrics'] = lyrics_df['lyrics'].apply(lambda l: ', '.join(l))
lyrics_input = ' '.join(lyrics_df['lyrics'].apply(lambda t: str(t)))

import plotly.express as px

def transition_dist_plot(phrase):
  text = standardization(lyrics_input)
  chars = list(text)
  dist_df = ngram_transition_matrix(chars, len(phrase) + 1).loc[tuple(phrase),:].reset_index()
  df = pd.DataFrame()
  df['char'] = dist_df.iloc[:,0]
  df['prob'] = dist_df.iloc[:,1]
  
  fig = px.histogram(df, x='char', y='prob', title="Transition Probabilities for Starting Phrase '" + phrase + "'")
  fig.write_html(phrase + 'dist.html')
  return fig

transition_dist_plot('you know ').show()

print(generate_song(lyrics_input, n=10))

import crochet
crochet.setup()
import pandas as pd
import json
from flask import Flask , render_template, jsonify, request, redirect, url_for
import scrapy
from scrapy.linkextractors import LinkExtractor
from bs4 import BeautifulSoup
import requests
import re
from scrapy.crawler import CrawlerRunner
from scrapy import signals
from scrapy.signalmanager import dispatcher
import time
from itertools import islice
import re
import numpy as np
import string
import os
# Importing our Scraping Function from the lyrics file
from lyrics1.lyrics1.spiders.lyrics import ImdbSpider

### stuff from last class
app = Flask(__name__)

output_data = []
crawl_runner = CrawlerRunner()

# By Deafult Flask will come into this when we run the file
@app.route('/')
def index():
	return render_template("index.html") # Returns index.html file in templates folder.


# After clicking the Submit Button FLASK will come into this
@app.route('/', methods=['POST'])
def submit():
    if request.method == 'POST':
        s = request.form['inp'] # Getting the Input 
        global inp
        inp = s
        
    # This will remove any existing file with the same name so that the scrapy will not append the data to any previous file.
    if os.path.exists("./webApp/app/file.json"): 
        os.remove("./webApp/app/file.json")

    return redirect(url_for('scrape')) # Passing to the Scrape function


@app.route("/scrape")
def scrape():

    scrape_with_crochet(form_inp=inp) # Passing that URL to our Scraping Function

    time.sleep(100)
    with open('file.json', 'w') as f:
        json.dump(output_data, f)
    
    lyrics_df = pd.read_json('./file.json')
    lyrics_df['lyrics'] = lyrics_df['lyrics'].apply(lambda l: ', '.join(l))

    lyrics_input = ' '.join(lyrics_df['lyrics'].apply(lambda t: str(t)))
    pd.set_option("display.max_rows", None, "display.max_columns", None)
    def consec_chars(chars, n):
        '''generates n-long tuples with a sliding window'''
        for i in range(len(chars) - (n-1)):
            yield chars[i:i+n]
    
    def ngram_transition_matrix(chars, n):
        '''Generates a transition matrix given a list of characters'''
        df = pd.DataFrame(consec_chars(chars, n), columns = [str(i) for i in range(1,n+1)])
        ngroup = list(df)[:n-1]
        counts = df.groupby(ngroup)[str(n)].value_counts()
        probs = (counts / counts.groupby(level=ngroup).sum()).unstack()
        return probs.fillna(0)
    
    def standardization(text):
        text = text.lower()
        text = re.sub(r'[\(\[].*?[\)\]]', '', text)
        text = re.sub(r',\s', ',', text)
        return text

    def next_word(tup, mat):
        '''sample next word given previous n-1 characters'''
        poss = mat.loc[tup,]
        return np.random.choice(list(mat), p = poss)

    def generate_block(title, start, length, mat, n):
        '''generates a new text of the given length'''
        start_tup = tuple(start)
        # get current MultiIndex (or regular index if doing bigram)
        curr = start_tup if n > 2 else start
        lyrics = '[' + title + ']' + '\n'
        lyrics += ''.join(start_tup)
        j = 0
        while j < length:
            nxt = next_word(curr, mat)
            lyrics += nxt + ('\n' if nxt == ',' else '')
            curr = nxt if n == 2 else (*curr[1:], nxt)
            if j < length - 1 or curr[-1] == ' ':
                j += 1
        
        lyrics = re.sub(',', ', ', lyrics)
        lyrics = re.sub(' ,', ',', lyrics)
        return re.sub('\n\s', '\n', lyrics) + '\n\n'
    
    def random_start(text, n):
        spaces = [i for i in range(len(text)) if text[i] == ' ']
        j = np.random.choice(spaces)
        return text[j+1:j+n]
    
    def generate_song(text, n=2, len_verse=500, len_chorus=100, num_verses=2):
        np.random.seed()
        text = standardization(text)
        starts = [random_start(text, n) for i in range(num_verses+1)]
        chars = list(text)
        mat = ngram_transition_matrix(chars, n)
        
        song = ""
        chorus = generate_block('Chorus', starts[0], len_chorus, mat, n)

        for i in range(num_verses):
            song += generate_block('Verse ' + str(i+1), starts[i+1], len_verse, mat, n)
            song += chorus
        
        return song
    
    word = generate_song(lyrics_input, n=10)

    return render_template('scrape.html', word = word) # Returns the scraped data after being running for 20 seconds.

@crochet.run_in_reactor
def scrape_with_crochet(form_inp):
    # This will connect to the dispatcher that will kind of loop the code between these two functions.
    dispatcher.connect(_crawler_result, signal=signals.item_scraped)
    
    # This will connect to the ReviewspiderSpider function in our scrapy file and after each yield will pass to the crawler_result function.
    eventual = crawl_runner.crawl(ImdbSpider, form_inp = form_inp)
    return eventual

#This will append the data to the output data list.
def _crawler_result(item, response, spider):
    output_data.append(dict(item))


if __name__== "__main__":
    app.run(debug=True)