import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
import numpy as np
import networkx as nx

nltk.download('punkt')
nltk.download('stopwords')

def sentence_similarity(sent1, sent2, stopwords=None):
    if stopwords is None:
        stopwords = []

    words1 = [word.lower() for word in sent1 if word.isalnum()]
    words2 = [word.lower() for word in sent2 if word.isalnum()]

    if len(words1) == 0 or len(words2) == 0:
        return 0

    common_words = len(set(words1) & set(words2))
    return common_words / (np.log(len(words1)) + np.log(len(words2)))

def build_similarity_matrix(sentences, stopwords):
    similarity_matrix = np.zeros((len(sentences), len(sentences)))

    for i in range(len(sentences)):
        for j in range(len(sentences)):
            if i == j:
                continue
            similarity_matrix[i][j] = sentence_similarity(sentences[i], sentences[j], stopwords)

    return similarity_matrix

def textrank_summarize(text, num_sentences=3):
    sentences = sent_tokenize(text)
    stopwords = set(stopwords.words("english"))
    similarity_matrix = build_similarity_matrix(sentences, stopwords)
    scores = nx.pagerank(nx.from_numpy_array(similarity_matrix))

    ranked_sentences = sorted(((scores[i], s) for i, s in enumerate(sentences)), reverse=True)

    summary = " ".join([sentence for score, sentence in ranked_sentences[:num_sentences]])
    return summary

# Example Usage
text = """
Text summarization is the process of distilling the most important information from a source to produce a shortened version for a particular audience or purpose.
The TextRank algorithm, inspired by Google's PageRank, assigns importance scores to sentences based on their similarity to other sentences in the text.
Extractive summarization methods select the most important sentences from the source text and concatenate them to form the summary.
One popular approach is the TextRank algorithm, which assigns importance scores to sentences based on their similarity to other sentences in the text.
"""

summary = textrank_summarize(text)
print("TextRank Summary:\n", summary)

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from nltk.tokenize import sent_tokenize
import numpy as np
import nltk

nltk.download('punkt')

def preprocess_text(text):
    # Tokenize the text into sentences
    sentences = sent_tokenize(text)
    return sentences

def lsa_summarize(text, num_sentences=3):
    # Preprocess the text
    sentences = preprocess_text(text)
    
    # Create a TF-IDF vectorizer
    tfidf_vectorizer = TfidfVectorizer(stop_words="english")
    
    # Fit and transform the text data
    tfidf_matrix = tfidf_vectorizer.fit_transform(sentences)
    
    # Perform LSA (Latent Semantic Analysis) using TruncatedSVD
    lsa = TruncatedSVD(n_components=num_sentences, random_state=42)
    lsa_matrix = lsa.fit_transform(tfidf_matrix)
    
    # Get the top sentences based on LSA components
    top_sentences_indices = np.argsort(np.sum(lsa_matrix, axis=1))[::-1][:num_sentences]
    top_sentences_indices.sort()
    
    # Generate the summary
    summary = ' '.join([sentences[i] for i in top_sentences_indices])
    
    return summary

# Example Usage
text = """
Text summarization is the process of distilling the most important information from a source to produce a shortened version for a particular audience or purpose.
Latent Semantic Analysis (LSA) is a technique that analyzes relationships between terms and concepts in a collection of texts based on the statistical occurrence of terms.
LSA identifies patterns in the relationships between terms and concepts and represents the texts in a lower-dimensional space to capture the underlying semantic structure.
The LSA algorithm transforms the original text data into a matrix representation and performs dimensionality reduction using techniques like Singular Value Decomposition (SVD).
"""

summary = lsa_summarize(text)
print("LSA Summary:\n", summary)

?@[\\]^_`{|}~\t\n”)
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)
padded_sequences = pad_sequences(sequences, padding=’post’)
return tokenizer, padded_sequences

def create_seq2seq_model(input_vocab_size, target_vocab_size, max_input_length, max_target_length, latent_dim):
# Encoder
encoder_inputs = Input(shape=(max_input_length,))
encoder_embedding = tf.keras.layers.Embedding(input_vocab_size, latent_dim, mask_zero=True)(encoder_inputs)
encoder_lstm = LSTM(latent_dim, return_state=True)
encoder_outputs, state_h, state_c = encoder_lstm(encoder_embedding)
encoder_states = [state_h, state_c]

# Dekoder
decoder_inputs = Input(shape=(max_target_length,))
decoder_embedding = tf.keras.layers.Embedding(target_vocab_size, latent_dim, mask_zero=True)(decoder_inputs)
decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)
decoder_dense = Dense(target_vocab_size, activation=’softmax’)
decoder_outputs = decoder_dense(decoder_outputs)

# Model
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model

# Przykładowe użycie
input_texts = [‘Text summarization is the process of distilling the most important information from a source to produce a shortened version for a particular audience or purpose.’,
‘Sequence-to-Sequence (Seq2Seq) models are a type of neural network architecture used in natural language processing tasks such as machine translation and text summarization.’]
target_texts = [‘Text summarization is the process of distilling important information from a source to produce a shortened version.’,
‘Seq2Seq models are neural network architectures used in NLP tasks like machine translation.’]

input_tokenizer, input_sequences = preprocess_text(input_texts)
target_tokenizer, target_sequences = preprocess_text(target_texts)

latent_dim = 256
input_vocab_size = len(input_tokenizer.word_index) + 1
target_vocab_size = len(target_tokenizer.word_index) + 1
max_input_length = input_sequences.shape[1]
max_target_length = target_sequences.shape[1]

model = create_seq2seq_model(input_vocab_size, target_vocab_size, max_input_length, max_target_length, latent_dim)
model.compile(optimizer=”adam”, loss=”sparse_categorical_crossentropy”, metrics=[‘accuracy’])

model.summary()
” data-lang=”text/x-python”>

import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, LSTM, Dense

def preprocess_text(texts):
    tokenizer = Tokenizer(filters="!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n")
    tokenizer.fit_on_texts(texts)
    sequences = tokenizer.texts_to_sequences(texts)
    padded_sequences = pad_sequences(sequences, padding='post')
    return tokenizer, padded_sequences

def create_seq2seq_model(input_vocab_size, target_vocab_size, max_input_length, max_target_length, latent_dim):
    # Encoder
    encoder_inputs = Input(shape=(max_input_length,))
    encoder_embedding = tf.keras.layers.Embedding(input_vocab_size, latent_dim, mask_zero=True)(encoder_inputs)
    encoder_lstm = LSTM(latent_dim, return_state=True)
    encoder_outputs, state_h, state_c = encoder_lstm(encoder_embedding)
    encoder_states = [state_h, state_c]

    # Decoder
    decoder_inputs = Input(shape=(max_target_length,))
    decoder_embedding = tf.keras.layers.Embedding(target_vocab_size, latent_dim, mask_zero=True)(decoder_inputs)
    decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)
    decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)
    decoder_dense = Dense(target_vocab_size, activation='softmax')
    decoder_outputs = decoder_dense(decoder_outputs)

    # Model
    model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
    return model

# Example Usage
input_texts = ['Text summarization is the process of distilling the most important information from a source to produce a shortened version for a particular audience or purpose.',
               'Sequence-to-Sequence (Seq2Seq) models are a type of neural network architecture used in natural language processing tasks such as machine translation and text summarization.']
target_texts = ['Text summarization is the process of distilling important information from a source to produce a shortened version.',
                'Seq2Seq models are neural network architectures used in NLP tasks like machine translation.']

input_tokenizer, input_sequences = preprocess_text(input_texts)
target_tokenizer, target_sequences = preprocess_text(target_texts)

latent_dim = 256
input_vocab_size = len(input_tokenizer.word_index) + 1
target_vocab_size = len(target_tokenizer.word_index) + 1
max_input_length = input_sequences.shape[1]
max_target_length = target_sequences.shape[1]

model = create_seq2seq_model(input_vocab_size, target_vocab_size, max_input_length, max_target_length, latent_dim)
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=['accuracy'])

model.summary()

from transformers import GPT2Tokenizer, GPT2LMHeadModel, pipeline

# Load pre-trained GPT-2 model and tokenizer
tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
model = GPT2LMHeadModel.from_pretrained("gpt2")

# Define a pipeline for text generation
text_generator = pipeline("text-generation", model=model, tokenizer=tokenizer)

# Example text
input_text = "Text summarization is the process of distilling the most important information from a source to produce a shortened version for a particular audience or purpose."

# Generate summary using GPT-2 model
summary = text_generator(input_text, max_length=100, num_return_sequences=1)[0]['generated_text']

print("GPT-2 Summary:\n", summary)

‘]stop_condition = False
decoded_summary = ”

while not stop_condition:
output_tokens, h, c = model.layers[3](decoder_input, initial_state=initial_state)
sampled_token_index = np.argmax(output_tokens[0, -1, :])
sampled_word = None
for word, index in tokenizer.word_index.items():
if index == sampled_token_index:
sampled_word = word
break
if sampled_word is None:
break
decoded_summary += sampled_word + ‘ ‘

if sampled_word == ‘‘ lub len(decoded_summary.split()) >= max_summary_length:
stop_condition = True

decoder_input = np.zeros((1, 1))
decoder_input[0, 0] = sampled_token_index
initial_state = [h, c]

return decoded_summary.strip()

# Generowanie podsumowania dla tekstu wejściowego
summary = generate_summary(input_text)
print(“Wygenerowane podsumowanie:\n”, summary)
” data-lang=”text/x-python”>

import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, LSTM, Dense, Embedding, Concatenate
from nltk.tokenize import sent_tokenize

# Example input text
input_text = """
Text summarization is the process of distilling the most important information from a source to produce a shortened version for a particular audience or purpose.
Extractive summarization methods select the most important sentences from the source text and concatenate them to form the summary.
Abstractive summarization, on the other hand, involves generating new sentences that capture the essence of the original text.
"""

# Preprocess the input text
sentences = sent_tokenize(input_text)
max_input_length = 100  # maximum number of words per sentence
max_summary_length = 20  # maximum number of words in the summary

# Tokenize sentences
tokenizer = Tokenizer()
tokenizer.fit_on_texts(sentences)
input_sequences = tokenizer.texts_to_sequences(sentences)
input_sequences = pad_sequences(input_sequences, maxlen=max_input_length, padding='post')

# Define Seq2Seq model for abstractive summarization
latent_dim = 256
vocab_size = len(tokenizer.word_index) + 1

# Encoder
encoder_inputs = Input(shape=(max_input_length,))
encoder_embedding = Embedding(vocab_size, latent_dim, mask_zero=True)(encoder_inputs)
encoder_lstm = LSTM(latent_dim, return_state=True)
encoder_outputs, state_h, state_c = encoder_lstm(encoder_embedding)
encoder_states = [state_h, state_c]

# Decoder
decoder_inputs = Input(shape=(None,))
decoder_embedding = Embedding(vocab_size, latent_dim, mask_zero=True)(decoder_inputs)
decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_embedding, initial_state=encoder_states)
decoder_dense = Dense(vocab_size, activation='softmax')
decoder_outputs = decoder_dense(decoder_outputs)

# Combined model
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)

# Compile model
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy")

# Train model (not shown in this example)

# Generate summary
def generate_summary(input_text):
    input_seq = tokenizer.texts_to_sequences([input_text])
    input_seq = pad_sequences(input_seq, maxlen=max_input_length, padding='post')
    initial_state = model.layers[1].states[0].numpy(), model.layers[1].states[1].numpy()
    decoder_input = np.zeros((1, 1))
    decoder_input[0, 0] = tokenizer.word_index['<start>']
    stop_condition = False
    decoded_summary = ''

    while not stop_condition:
        output_tokens, h, c = model.layers[3](decoder_input, initial_state=initial_state)
        sampled_token_index = np.argmax(output_tokens[0, -1, :])
        sampled_word = None
        for word, index in tokenizer.word_index.items():
            if index == sampled_token_index:
                sampled_word = word
                break
        if sampled_word is None:
            break
        decoded_summary += sampled_word + ' '

        if sampled_word == '<end>' or len(decoded_summary.split()) >= max_summary_length:
            stop_condition = True

        decoder_input = np.zeros((1, 1))
        decoder_input[0, 0] = sampled_token_index
        initial_state = [h, c]

    return decoded_summary.strip()

# Generate summary for the input text
summary = generate_summary(input_text)
print("Generated Summary:\n", summary)