import nltk
from nltk.corpus import stopwords

try:
    nltk.data.find('corpora/stopwords')
except LookupError:
    nltk.download('stopwords')

def remove_stopwords(text):
    """
    Remove stopwords using NLTK's stopword list
    
    Args:
        text (str): Input text
        
    Returns:
        str: Cleaned text with stopwords removed
    """
    stop_words = set(stopwords.words('english'))
    words = text.lower().split()
    return ' '.join([word for word in words if word not in stop_words])

def is_exact_match(ngram, sentences):
    """
    Check if the given n-gram has an exact match in all sentences
    
    Args:
        ngram (str): The n-gram to search for
        sentences (list): List of sentences to search in
        
    Returns:
        bool: True if n-gram has exact match in all sentences, False otherwise
    """
    sentence_ngrams = []
    for sentence in sentences:
        words = sentence.split()
        current_ngrams = []
        n = len(ngram.split())
        
        for i in range(len(words) - n + 1):
            current_ngram = " ".join(words[i:i+n])
            current_ngrams.append(current_ngram)
            
        sentence_ngrams.append(set(current_ngrams))
    
    return all(ngram in sent_ngrams for sent_ngrams in sentence_ngrams)

def is_substring_of_any(ngram, common_ngrams):
    """
    Check if the given n-gram is an exact substring of any previously found common n-grams
    
    Args:
        ngram (str): The n-gram to check
        common_ngrams (list): List of previously found common n-grams
        
    Returns:
        bool: True if ngram is a substring of any common_ngrams, False otherwise
    """
    ngram_words = ngram.split()
    for common_gram in common_ngrams:
        common_words = common_gram.split()
        for i in range(len(common_words) - len(ngram_words) + 1):
            if " ".join(common_words[i:i+len(ngram_words)]) == ngram:
                return True
    return False

def find_filtered_ngrams(sentences):
    """
    Find all n-grams that have exact matches across all sentences,
    excluding those that are part of larger common n-grams
    
    Args:
        sentences (list): List of sentences to analyze
    
    Returns:
        list: List of all common n-grams in order of their appearance in the first sentence
    """
    # First, remove stopwords from all sentences
    cleaned_sentences = [remove_stopwords(sentence) for sentence in sentences]
    
    words = cleaned_sentences[0].split()
    max_n = len(words)
    all_common_ngrams = []
    
    for n in range(max_n, 0, -1):
        for i in range(len(words) - n + 1):
            ngram = " ".join(words[i:i+n])
            
            if is_exact_match(ngram, cleaned_sentences) and not is_substring_of_any(ngram, all_common_ngrams):
                all_common_ngrams.append(ngram)
    
    return all_common_ngrams

def find_relative_order(sentence, common_ngrams):
    sentence = sentence.lower()
    ngram_positions = {}
    
    for ngram in common_ngrams:
        ngram_lower = ngram.lower()
        if ngram_lower in sentence:
            position = sentence.index(ngram_lower)
            ngram_positions[ngram] = position
    
    sorted_ngrams = sorted(ngram_positions.items(), key=lambda x: x[1])
    
    result = [(i + 1, ngram) for i, (ngram, _) in enumerate(sorted_ngrams)]
    
    return result


def find_non_melting_points(sent_list):
    
    # Find filtered n-grams
    common_ngrams = find_filtered_ngrams(sent_list)
    
    def remove_punctuation(common_ngrams):
      punctuation = ".?!.;,:'\"()[]{}-–—...+/\\*^|@#%&_~`"
      for item in common_ngrams:
        if item in punctuation:
          common_ngrams.remove(item)
      return common_ngrams

    final_list = remove_punctuation(common_ngrams)
    sentence = sent_list[0]
    non_melting_points = find_relative_order(sentence, final_list)

    return non_melting_points


# Example usage
# from paraphraser import generate_paraphrase
# from twokenize import tokenize_sentences

# sentences = tokenize_sentences(generate_paraphrase("I'm very skeptical that the next \"great writer\" will be a robot or that they'll be much more effective at expressing the subtleties and depths of human thought than a human is."))
# non_melting_points = find_non_melting_points(sentences)

# print(non_melting_points)