# tools.pyimport osfrom crawl4ai import WebCrawlerfrom crawl4ai.extraction_strategy import LLMExtractionStrategyfrom pydantic import BaseModel, Fieldfrom praisonai_tools import BaseTool
class ModelFee(BaseModel):llm_model_name: str = Field(..., description="Name of the model.")input_fee: str = Field(..., description="Fee for input token for the model.")output_fee: str = Field(..., description="Fee for output token for the model.")
class ModelFeeTool(BaseTool):name: str = "ModelFeeTool"description: str = "Extracts model fees for input and output tokens from the given pricing page."
def _run(self, url: str):crawler = WebCrawler()crawler.warmup()
result = crawler.run(url=url,word_count_threshold=1,extraction_strategy= LLMExtractionStrategy(provider="openai/gpt-4o",api_token=os.getenv('OPENAI_API_KEY'), schema=ModelFee.schema(),extraction_type="schema",instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens. Do not miss any models in the entire content. One extracted model JSON format should look like this: {"model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens"}."""),bypass_cache=True,)return result.extracted_content
if __name__ == "__main__":# Test the ModelFeeTooltool = ModelFeeTool()url = "https://www.openai.com/pricing"result = tool.run(url)print(result)

framework: crewaitopic: extract model pricing from websitesroles:web_scraper:backstory: An expert in web scraping with a deep understanding of extracting structureddata from online sources. https://openai.com/api/pricing/ https://www.anthropic.com/pricing https://cohere.com/pricinggoal: Gather model pricing data from various websitesrole: Web Scrapertasks:scrape_model_pricing:description: Scrape model pricing information from the provided list of websites.expected_output: Raw HTML or JSON containing model pricing data.tools:- 'ModelFeeTool'data_cleaner:backstory: Specialist in data cleaning, ensuring that all collected data is accurateand properly formatted.goal: Clean and organize the scraped pricing datarole: Data Cleanertasks:clean_pricing_data:description: Process the raw scraped data to remove any duplicates and inconsistencies,and convert it into a structured format.expected_output: Cleaned and organized JSON or CSV file with model pricingdata.tools:- ''data_analyzer:backstory: Data analysis expert focused on deriving actionable insights from structureddata.goal: Analyze the cleaned pricing data to extract insightsrole: Data Analyzertasks:analyze_pricing_data:description: Analyze the cleaned data to extract trends, patterns, and insightson model pricing.expected_output: Detailed report summarizing model pricing trends and insights.tools:- ''