import nltkfrom nltk.corpus import stopwordsfrom nltk.tokenize import word_tokenizefrom nltk.stem import WordNetLemmatizernltk.download('stopwords')
def preprocess_text(text):    tokens = word_tokenize(text)    tokens = [word.lower() for word in tokens]    tokens = [word for word in tokens if word.isalnum()]    # remove stop words    stop_words = set(stopwords.words('english'))    tokens = [word for word in tokens if word not in stop_words]    # Lemmatize     lemmatizer = WordNetLemmatizer()    tokens = [lemmatizer.lemmatize(word) for word in tokens]
    return ' '.join(tokens)

from tensorflow.keras.preprocessing.text import one_hotfrom tensorflow.keras.preprocessing.sequence import pad_sequences
documents = [    "The quick brown fox jumps over the lazy dog",    "A journey of a thousand miles begins with a single step",    "To be or not to be that is the question"]
vocab_size=50 integer_encoded=[]for i,doc in enumerate(documents):    integer_encoded.append(one_hot(doc,50))    print("The encoding for text",i+1," is : ",one_hot(doc,50))

# length of maximum documentmaxlen = max([len(word_tokenize(text)) for text in documents])maxlen
# OUTPUT11

# To create embeddings, all documents must have the same length. hence we can pad to ensure uniformity.pad_corp = pad_sequences(integer_encoded, maxlen=maxlen,padding='post',value=0.0)print("No of padded documents: ",len(pad_corp))
# OUTPUTNo of padded documents:  3

for i,doc in enumerate(pad_corp):     print("The padded encoding for text",i+1," is : ",doc)

from sklearn.feature_extraction.text import TfidfVectorizer
preprocessed_text = [preprocess_text(doc) for doc in documents]
tfidf = TfidfVectorizer()tfidf_matrix = tfidf.fit_transform(preprocessed_text)print("Vocabulary:", tfidf.vocabulary_)tfidf_matrix.toarray()

from gensim.models import Word2Vecfrom nltk.tokenize import word_tokenizeimport numpy as np
tokenized_docs = [word_tokenize(doc.lower()) for doc in documents]w2v_model = Word2Vec(sentences=tokenized_docs, vector_size=100, window=5, min_count=1, workers=4)
def get_sentence_embedding(sentence, model):    words = word_tokenize(sentence.lower())    word_vectors = [model.wv[word] for word in words if word in model.wv]    return np.mean(word_vectors, axis=0) if word_vectors else np.zeros(model.vector_size)
print("Word2Vec Sentence Embeddings:")for doc in documents:    sentence_embedding = get_sentence_embedding(doc, w2v_model)    print(f"Sentence: {doc}.\n Embedding: {sentence_embedding}")

from gensim.models import FastText
fasttext_model = FastText(sentences=tokenized_docs, vector_size=100, window=5, min_count=1, workers=4)
def get_sentence_embedding_fasttext(sentence, model):    words = word_tokenize(sentence.lower())    word_vectors = [model.wv[word] for word in words if word in model.wv]    return np.mean(word_vectors, axis=0) if word_vectors else np.zeros(model.vector_size)
print("FastText Sentence Embeddings:")for doc in documents:    sentence_embedding = get_sentence_embedding_fasttext(doc, fasttext_model)    print(f"Sentence: {doc}. \nEmbedding: {sentence_embedding}")

from transformers import BertTokenizer, BertModelimport torch
# Load pre-trained BERT tokenizer and modeltokenizer = BertTokenizer.from_pretrained('bert-base-uncased')model = BertModel.from_pretrained('bert-base-uncased')
def get_sentence_embedding_bert(sentence, model, tokenizer):    inputs = tokenizer(sentence, return_tensors='pt')    outputs = model(**inputs)        # Get sentence embedding by averaging token embeddings    sentence_embedding = torch.mean(outputs.last_hidden_state, dim=1)    return sentence_embedding.squeeze().detach().numpy()
print("BERT Sentence Embeddings:")for doc in documents:    sentence_embedding = get_sentence_embedding_bert(doc, model, tokenizer)    print(f"Sentence: {doc}. \nEmbedding: {sentence_embedding}")