from rank_bm25 import BM25Okapifrom nltk.tokenize import word_tokenize
# Sample documentsdocuments = ["The cat sat on the mat.","The dog barked at the moon.","The sun is shining bright."]
# Tokenize the documentstokenized_corpus = [word_tokenize(doc.lower()) for doc in documents]
# Initialize BM25bm25 = BM25Okapi(tokenized_corpus)
# Queryquery = "cat on mat"tokenized_query = word_tokenize(query.lower())
# Retrieve BM25 resultsbm25_scores = bm25.get_scores(tokenized_query)bm25_results = bm25.get_top_n(tokenized_query, documents, n=3)
print("BM25 Results: ", bm25_results)

from transformers import AutoTokenizer, AutoModelimport torchimport faiss
# Load a pre-trained model and tokenizer for embedding creationtokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")model = AutoModel.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
# Embed documentsdef embed_texts(texts):inputs = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")with torch.no_grad():embeddings = model(**inputs).last_hidden_state.mean(dim=1)return embeddings
# Generate embeddings for the documentsdoc_embeddings = embed_texts(documents).numpy()
# Generate an embedding for the queryquery_embedding = embed_texts([query]).numpy()
# Use FAISS to index and search the documents based on embeddingsindex = faiss.IndexFlatL2(doc_embeddings.shape[1])index.add(doc_embeddings)
# Search for the top 3 most similar documents_, dense_results_idx = index.search(query_embedding, k=3)dense_results = [documents[idx] for idx in dense_results_idx[0]]
print("Dense Retrieval Results: ", dense_results)

import numpy as np
# Normalize BM25 and Dense retrieval scoresbm25_scores = np.array(bm25_scores)bm25_scores_normalized = bm25_scores / np.max(bm25_scores)
dense_scores = np.linalg.norm(query_embedding - doc_embeddings, axis=1)dense_scores_normalized = 1 - (dense_scores / np.max(dense_scores))# Convert distances to similarity
# Combine the normalized scores (you can adjust the weights as needed)combined_scores = 0.5 * bm25_scores_normalized + 0.5 * dense_scores_normalized
# Get the top documents based on combined scorestop_idx = combined_scores.argsort()[::-1]hybrid_results = [documents[i] for i in top_idx[:3]]
print("Hybrid Search Results: ", hybrid_results)