diff --git a/CHANGELOG.md b/CHANGELOG.md index e4bd45e3..00fac3e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,75 @@ +## [1.59.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.58.0...v1.59.0) (2025-06-24) + + +### Features + +* removed sposnsors ([288c69a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/288c69a862f34b999db476e669ff97c00afacde3)) + +## [1.58.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.57.0...v1.58.0) (2025-06-21) + + +### Features + +* add new oss link ([0c2481f](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0c2481fffebca355e542ae420ee1bf4cade8e5e3)) + + +### Docs + +* add links to other language versions of README ([07dec35](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/07dec35f1bf95842ee55b17796bb45f2db0f44b3)) + +## [1.57.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.56.0...v1.57.0) (2025-06-13) + + +### Features + +* add markdownify endpoint ([7340375](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/73403755da1e4c3065e91d834c59f6d8c1825763)) + +## [1.56.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.55.0...v1.56.0) (2025-06-13) + + +### Features + +* add scrapegraphai integration ([94e9ebd](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/94e9ebd28061f8313bb23074b4db3406cf4db0c9)) + +## [1.55.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.54.1...v1.55.0) (2025-06-07) + + +### Features + +* add adv ([cd29791](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/cd29791894325c54f1dec1d2a5f6456800beb63e)) +* update logs ([8c54162](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/8c541620879570c46f32708c7e488e9a4ca0ea3e)) + +## [1.54.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.54.0...v1.54.1) (2025-06-06) + + +### Bug Fixes + +* bug on generate answer node ([e846a14](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/e846a1415506a58f7bc8b76ac56ba0b6413178ba)) + +## [1.54.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.53.0...v1.54.0) (2025-06-06) + + +### Features + +* add grok integration ([0c476a4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0c476a4a7bbbec3883f505cd47bcffdcd2d9e5fd)) + + +### Bug Fixes + +* grok integration and add new grok models ([3f18272](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3f1827274c60a2729233577666d2fa446c48c4ba)) + + +### chore + +* enhanced a readme ([68bb34c](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/68bb34cc5e63b8a1d5acc61b9b61f9ea716a2a51)) + + +### CI + +* **release:** 1.52.0-beta.1 [skip ci] ([7adb0f1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/7adb0f1df1efc4e6ada1134f6e53e4d6b072a608)) +* **release:** 1.52.0-beta.2 [skip ci] ([386b46a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/386b46a8692c8c18000bb071fc8f312adc3ad05e)) +* **release:** 1.54.0-beta.1 [skip ci] ([77d4432](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/77d44321a1d41e10ac6aa13b526a49e718bd7c5d)) + ## [1.54.0-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.53.0...v1.54.0-beta.1) (2025-06-06) diff --git a/README.md b/README.md index 5598f140..53f30484 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## 🚀 **Looking for an even faster and simpler way to scrape at scale (only 5 lines of code)? ** Check out our enhanced version at [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&ut#m_content=top_banner)! 🚀 +## 🚀 **Looking for an even faster and simpler way to scrape at scale (only 5 lines of code)?** Check out our enhanced version at [**ScrapeGraphAI.com**](https://scrapegraphai.com/?utm_source=github&utm_medium=readme&utm_campaign=oss_cta&ut#m_content=top_banner)! 🚀 --- @@ -7,6 +7,10 @@ [English](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/README.md) | [中文](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/docs/chinese.md) | [日本語](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/docs/japanese.md) | [한국어](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/docs/korean.md) | [Русский](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/docs/russian.md) | [Türkçe](https://github.com/VinciGit00/Scrapegraph-ai/blob/main/docs/turkish.md) +| [Deutsch](https://www.readme-i18n.com/ScrapeGraphAI/Scrapegraph-ai?lang=de) +| [Español](https://www.readme-i18n.com/ScrapeGraphAI/Scrapegraph-ai?lang=es) +| [français](https://www.readme-i18n.com/ScrapeGraphAI/Scrapegraph-ai?lang=fr) +| [Português](https://www.readme-i18n.com/ScrapeGraphAI/Scrapegraph-ai?lang=pt) [![Downloads](https://img.shields.io/pepy/dt/scrapegraphai?style=for-the-badge)](https://pepy.tech/project/scrapegraphai) @@ -39,7 +43,7 @@ You can find more informations at the following [link](https://scrapegraphai.com - **API**: [Documentation](https://docs.scrapegraphai.com/introduction) - **SDKs**: [Python](https://docs.scrapegraphai.com/sdks/python), [Node](https://docs.scrapegraphai.com/sdks/javascript) - **LLM Frameworks**: [Langchain](https://docs.scrapegraphai.com/integrations/langchain), [Llama Index](https://docs.scrapegraphai.com/integrations/llamaindex), [Crew.ai](https://docs.scrapegraphai.com/integrations/crewai), [CamelAI](https://github.com/camel-ai/camel) -- **Low-code Frameworks**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [LangFlow](https://www.langflow.org) +- **Low-code Frameworks**: [Pipedream](https://pipedream.com/apps/scrapegraphai), [Bubble](https://bubble.io/plugin/scrapegraphai-1745408893195x213542371433906180), [Zapier](https://zapier.com/apps/scrapegraphai/integrations), [n8n](http://localhost:5001/dashboard), [LangFlow](https://www.langflow.org), [Dify](https://dify.ai) - **MCP server**: [Link](https://smithery.ai/server/@ScrapeGraphAI/scrapegraph-mcp) ## 🚀 Quick install @@ -183,22 +187,6 @@ We offer SDKs in both Python and Node.js, making it easy to integrate into your The Official API Documentation can be found [here](https://docs.scrapegraphai.com/). -## 🏆 Sponsors -
- - Browserbase - - - SerpAPI - - - Stats - - - Stats - -
- ## 📈 Telemetry We collect anonymous usage metrics to enhance our package's quality and user experience. The data helps us prioritize improvements and ensure compatibility. If you wish to opt-out, set the environment variable SCRAPEGRAPHAI_TELEMETRY_ENABLED=false. For more information, please refer to the documentation [here](https://scrapegraph-ai.readthedocs.io/en/latest/scrapers/telemetry.html). @@ -235,3 +223,5 @@ ScrapeGraphAI is licensed under the MIT License. See the [LICENSE](https://githu - ScrapeGraphAI is meant to be used for data exploration and research purposes only. We are not responsible for any misuse of the library. Made with ❤️ by [ScrapeGraph AI](https://scrapegraphai.com) + +[Scarf tracking](https://static.scarf.sh/a.png?x-pxid=102d4b8c-cd6a-4b9e-9a16-d6d141b9212d) diff --git a/docs/assets/scrapedo.png b/docs/assets/scrapedo.png deleted file mode 100644 index d15ea04e..00000000 Binary files a/docs/assets/scrapedo.png and /dev/null differ diff --git a/docs/assets/scrapeless.png b/docs/assets/scrapeless.png deleted file mode 100644 index 6e81efbc..00000000 Binary files a/docs/assets/scrapeless.png and /dev/null differ diff --git a/docs/assets/serp_api_logo.png b/docs/assets/serp_api_logo.png deleted file mode 100644 index ff2f1b01..00000000 Binary files a/docs/assets/serp_api_logo.png and /dev/null differ diff --git a/docs/assets/transparent_stat.png b/docs/assets/transparent_stat.png deleted file mode 100644 index d4f3c5fc..00000000 Binary files a/docs/assets/transparent_stat.png and /dev/null differ diff --git a/examples/markdownify/.env.example b/examples/markdownify/.env.example new file mode 100644 index 00000000..8f19deb6 --- /dev/null +++ b/examples/markdownify/.env.example @@ -0,0 +1 @@ +SCRAPEGRAPH_API_KEY=your SCRAPEGRAPH_API_KEY \ No newline at end of file diff --git a/examples/markdownify/markdownify_scrapegraphai.py b/examples/markdownify/markdownify_scrapegraphai.py new file mode 100644 index 00000000..113fe5cd --- /dev/null +++ b/examples/markdownify/markdownify_scrapegraphai.py @@ -0,0 +1,35 @@ +""" +Example script demonstrating the markdownify functionality +""" + +import os +from dotenv import load_dotenv +from scrapegraph_py import Client +from scrapegraph_py.logger import sgai_logger + +def main(): + # Load environment variables + load_dotenv() + + # Set up logging + sgai_logger.set_logging(level="INFO") + + # Initialize the client + api_key = os.getenv("SCRAPEGRAPH_API_KEY") + if not api_key: + raise ValueError("SCRAPEGRAPH_API_KEY environment variable not found") + sgai_client = Client(api_key=api_key) + + # Example 1: Convert a website to Markdown + print("Example 1: Converting website to Markdown") + print("-" * 50) + response = sgai_client.markdownify( + website_url="https://example.com" + ) + print("Markdown output:") + print(response["result"]) # Access the result key from the dictionary + print("\nMetadata:") + print(response.get("metadata", {})) # Use get() with default value + print("\n" + "=" * 50 + "\n") +if __name__ == "__main__": + main() diff --git a/examples/markdownify/readme.md b/examples/markdownify/readme.md new file mode 100644 index 00000000..46c506df --- /dev/null +++ b/examples/markdownify/readme.md @@ -0,0 +1,75 @@ +# Markdownify Graph Example + +This example demonstrates how to use the Markdownify graph to convert HTML content to Markdown format. + +## Features + +- Convert HTML content to clean, readable Markdown +- Support for both URL and direct HTML input +- Maintains formatting and structure of the original content +- Handles complex HTML elements and nested structures + +## Usage + +```python +from scrapegraphai import Client +from scrapegraphai.logger import sgai_logger + +# Set up logging +sgai_logger.set_logging(level="INFO") + +# Initialize the client +sgai_client = Client(api_key="your-api-key") + +# Example 1: Convert a website to Markdown +response = sgai_client.markdownify( + website_url="https://example.com" +) +print(response.markdown) + +# Example 2: Convert HTML content directly +html_content = """ +
+

Hello World

+

This is a test paragraph.

+
+""" +response = sgai_client.markdownify( + html_content=html_content +) +print(response.markdown) +``` + +## Parameters + +The `markdownify` method accepts the following parameters: + +- `website_url` (str, optional): The URL of the website to convert to Markdown +- `html_content` (str, optional): Direct HTML content to convert to Markdown + +Note: You must provide either `website_url` or `html_content`, but not both. + +## Response + +The response object contains: + +- `markdown` (str): The converted Markdown content +- `metadata` (dict): Additional information about the conversion process + +## Error Handling + +The graph handles various edge cases: + +- Invalid URLs +- Malformed HTML +- Network errors +- Timeout issues + +If an error occurs, it will be logged and raised with appropriate error messages. + +## Best Practices + +1. Always provide a valid URL or well-formed HTML content +2. Use appropriate logging levels for debugging +3. Handle the response appropriately in your application +4. Consider rate limiting for large-scale conversions diff --git a/examples/search_graph/scrapegraphai/.env.example b/examples/search_graph/scrapegraphai/.env.example new file mode 100644 index 00000000..8f19deb6 --- /dev/null +++ b/examples/search_graph/scrapegraphai/.env.example @@ -0,0 +1 @@ +SCRAPEGRAPH_API_KEY=your SCRAPEGRAPH_API_KEY \ No newline at end of file diff --git a/examples/search_graph/scrapegraphai/readme.md b/examples/search_graph/scrapegraphai/readme.md new file mode 100644 index 00000000..e69de29b diff --git a/examples/search_graph/scrapegraphai/searchscraper_scrapegraphai.py b/examples/search_graph/scrapegraphai/searchscraper_scrapegraphai.py new file mode 100644 index 00000000..77a655f9 --- /dev/null +++ b/examples/search_graph/scrapegraphai/searchscraper_scrapegraphai.py @@ -0,0 +1,83 @@ +""" +Example implementation of search-based scraping using Scrapegraph AI. +This example demonstrates how to use the searchscraper to extract information from the web. +""" + +import os +from typing import Dict, Any +from dotenv import load_dotenv +from scrapegraph_py import Client +from scrapegraph_py.logger import sgai_logger + +def format_response(response: Dict[str, Any]) -> None: + """ + Format and print the search response in a readable way. + + Args: + response (Dict[str, Any]): The response from the search API + """ + print("\n" + "="*50) + print("SEARCH RESULTS") + print("="*50) + + # Print request ID + print(f"\nRequest ID: {response['request_id']}") + + # Print number of sources + urls = response.get('reference_urls', []) + print(f"\nSources Processed: {len(urls)}") + + # Print the extracted information + print("\nExtracted Information:") + print("-"*30) + if isinstance(response['result'], dict): + for key, value in response['result'].items(): + print(f"\n{key.upper()}:") + if isinstance(value, list): + for item in value: + print(f" • {item}") + else: + print(f" {value}") + else: + print(response['result']) + + # Print source URLs + if urls: + print("\nSources:") + print("-"*30) + for i, url in enumerate(urls, 1): + print(f"{i}. {url}") + print("\n" + "="*50) + +def main(): + # Load environment variables + load_dotenv() + + # Get API key + api_key = os.getenv("SCRAPEGRAPH_API_KEY") + if not api_key: + raise ValueError("SCRAPEGRAPH_API_KEY not found in environment variables") + + # Configure logging + sgai_logger.set_logging(level="INFO") + + # Initialize client + sgai_client = Client(api_key=api_key) + + try: + # Basic search scraper example + print("\nSearching for information...") + + search_response = sgai_client.searchscraper( + user_prompt="Extract webpage information" + ) + format_response(search_response) + + except Exception as e: + print(f"\nError occurred: {str(e)}") + finally: + # Always close the client + sgai_client.close() + +if __name__ == "__main__": + main() diff --git a/examples/smart_scraper_graph/README.md b/examples/smart_scraper_graph/README.md deleted file mode 100644 index d5f0d564..00000000 --- a/examples/smart_scraper_graph/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Smart Scraper Example - -This example demonstrates how to use Scrapegraph-ai for intelligent web scraping with automatic content detection and extraction. - -## Features - -- Intelligent content detection -- Automatic data extraction -- Content classification -- Clean data output - -## Setup - -1. Install required dependencies -2. Copy `.env.example` to `.env` -3. Configure your OpenAI API key in the `.env` file - -## Usage - -```python -from scrapegraphai.graphs import SmartScraperGraph - -graph = SmartScraperGraph() -results = graph.scrape("https://example.com") -``` - -## Environment Variables - -Required environment variables: -- `OPENAI_API_KEY`: Your OpenAI API key diff --git a/examples/smart_scraper_graph/ollama/smart_scraper_ollama.py b/examples/smart_scraper_graph/ollama/smart_scraper_ollama.py index 9642b3f8..ac3ee8a0 100644 --- a/examples/smart_scraper_graph/ollama/smart_scraper_ollama.py +++ b/examples/smart_scraper_graph/ollama/smart_scraper_ollama.py @@ -11,7 +11,7 @@ graph_config = { "llm": { - "model": "ollama/llama3.2:3b", + "model": "ollama/llama3.2", "temperature": 0, # "base_url": "http://localhost:11434", # set ollama URL arbitrarily "model_tokens": 4096, @@ -24,7 +24,7 @@ # Create the SmartScraperGraph instance and run it # ************************************************ smart_scraper_graph = SmartScraperGraph( - prompt="Find some information about what does the company do and the list of founders.", + prompt="Find some information about the founders.", source="https://scrapegraphai.com/", config=graph_config, ) diff --git a/examples/smart_scraper_graph/scrapegraphai/.env.example b/examples/smart_scraper_graph/scrapegraphai/.env.example new file mode 100644 index 00000000..8f19deb6 --- /dev/null +++ b/examples/smart_scraper_graph/scrapegraphai/.env.example @@ -0,0 +1 @@ +SCRAPEGRAPH_API_KEY=your SCRAPEGRAPH_API_KEY \ No newline at end of file diff --git a/examples/smart_scraper_graph/scrapegraphai/readme.md b/examples/smart_scraper_graph/scrapegraphai/readme.md new file mode 100644 index 00000000..9be52a33 --- /dev/null +++ b/examples/smart_scraper_graph/scrapegraphai/readme.md @@ -0,0 +1,148 @@ +# Smart Scraper Examples with Scrapegraph AI + +This repository contains examples demonstrating how to use Scrapegraph AI's powerful web scraping capabilities to transform websites into structured data using natural language prompts. + +## About Scrapegraph AI + +[Scrapegraph AI](https://scrapegraphai.com) is a powerful web scraping API that transforms any website into structured data for AI agents and analytics. It's built specifically for AI agents and LLMs, featuring natural language instructions and structured JSON output. + +Key features: +- Universal data extraction from any website +- Intelligent processing with advanced AI +- Lightning-fast setup with official SDKs +- Enterprise-ready with automatic proxy rotation +- Seamless integration with RAG systems + +## Examples Included + +### 1. Smart Scraper +The `smartscraper_scrapegraphai.py` example demonstrates how to extract structured data from a single website using natural language prompts. + +### 2. Search Scraper +The `searchscraper_scrapegraphai.py` example shows how to: +- Search the internet for relevant information +- Extract structured data from multiple sources +- Merge and analyze information from different websites +- Get comprehensive answers to complex queries + +## Prerequisites + +- Python 3.7+ +- pip (Python package manager) + +## Installation + +1. Clone the repository: +```bash +git clone https://github.com/yourusername/Scrapegraph-ai.git +cd Scrapegraph-ai +``` + +2. Install required dependencies: +```bash +pip install -r requirements.txt +``` + +3. Create a `.env` file in the `examples/smart_scraper_graph` directory with: +```env +SCRAPEGRAPH_API_KEY=your-api-key-here +``` + +## Usage + +### Smart Scraper Example +```bash +python smartscraper_scrapegraphai.py +``` + +### Search Scraper Example +```bash +python searchscraper_scrapegraphai.py +``` + +## Example Outputs + +### Smart Scraper Output +```python +Request ID: abc123... +Result: { + "founders": [ + { + "name": "Marco Vinciguerra", + "role": "Founder & Software Engineer", + "bio": "LinkedIn profile of Marco Vinciguerra" + }, + { + "name": "Lorenzo Padoan", + "role": "Founder & CEO", + "bio": "LinkedIn profile of Lorenzo Padoan" + } + ] +} +Reference URLs: ["https://scrapegraphai.com/about"] +``` + +### Search Scraper Output +```python +Request ID: xyz789... +Number of sources processed: 3 + +Extracted Information: +{ + "features": [ + "Universal data extraction", + "Intelligent processing with AI", + "Lightning-fast setup", + "Enterprise-ready with proxy rotation" + ], + "benefits": [ + "Perfect for AI agents and LLMs", + "Natural language instructions", + "Structured JSON output", + "Seamless RAG integration" + ] +} + +Sources: +1. https://scrapegraphai.com +2. https://scrapegraphai.com/features +3. https://scrapegraphai.com/docs +``` + +## Features Demonstrated + +- Environment variable configuration +- API client initialization +- Smart scraping with natural language prompts +- Search-based scraping across multiple sources +- Error handling and response processing +- Secure credential management + +## Pricing and Credits + +Scrapegraph AI offers various pricing tiers: +- Free: 50 credits included +- Starter: $20/month, 5,000 credits +- Growth: $100/month, 40,000 credits +- Pro: $500/month, 250,000 credits +- Enterprise: Custom solutions + +Service costs: +- Smart Scraper: 10 credits per webpage +- Search Scraper: 30 credits per query + +## Support and Resources + +- [Official Documentation](https://scrapegraphai.com/docs) +- [API Status](https://scrapegraphai.com/status) +- Contact: contact@scrapegraphai.com + +## Security Notes + +- Never commit your `.env` file to version control +- Keep your API key secure +- Use environment variables for sensitive credentials + +## License + +This example is provided under the same license as Scrapegraph AI. See the [Terms of Service](https://scrapegraphai.com/terms) for more information. diff --git a/examples/smart_scraper_graph/scrapegraphai/smartscraper_scrapegraphai.py b/examples/smart_scraper_graph/scrapegraphai/smartscraper_scrapegraphai.py new file mode 100644 index 00000000..989c7d62 --- /dev/null +++ b/examples/smart_scraper_graph/scrapegraphai/smartscraper_scrapegraphai.py @@ -0,0 +1,45 @@ +""" +Example implementation using scrapegraph-py client directly. +""" + +import os +from dotenv import load_dotenv +from scrapegraph_py import Client +from scrapegraph_py.logger import sgai_logger + +def main(): + # Load environment variables from .env file + load_dotenv() + + # Get API key from environment variables + api_key = os.getenv("SCRAPEGRAPH_API_KEY") + if not api_key: + raise ValueError("SCRAPEGRAPH_API_KEY non trovato nelle variabili d'ambiente") + + # Set up logging + sgai_logger.set_logging(level="INFO") + + # Initialize the client with API key from environment + sgai_client = Client(api_key=api_key) + + try: + # SmartScraper request + response = sgai_client.smartscraper( + website_url="https://scrapegraphai.com", + user_prompt="Extract the founders' informations" + ) + + # Print the response + print(f"Request ID: {response['request_id']}") + print(f"Result: {response['result']}") + if response.get('reference_urls'): + print(f"Reference URLs: {response['reference_urls']}") + + except Exception as e: + print(f"Error occurred: {str(e)}") + finally: + # Always close the client + sgai_client.close() + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 80d76fa7..8e026b34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "scrapegraphai" -version = "1.54.0b1" +version = "1.59.0" description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." authors = [ @@ -31,6 +31,7 @@ dependencies = [ "jsonschema>=4.23.0", "duckduckgo-search>=7.2.1", "pydantic>=2.10.2", + "scrapegraph-py>=0.1.0", ] readme = "README.md" diff --git a/scrapegraphai/__init__.py b/scrapegraphai/__init__.py index 52b4d951..4136a158 100644 --- a/scrapegraphai/__init__.py +++ b/scrapegraphai/__init__.py @@ -1,3 +1,8 @@ """ __init__.py file for scrapegraphai folder """ + +from .utils.logging import get_logger, set_verbosity_info + +logger = get_logger(__name__) +set_verbosity_info() diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index 90f6efc2..4af185c1 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -5,6 +5,7 @@ import asyncio import uuid import warnings +import time from abc import ABC, abstractmethod from typing import Optional, Type @@ -14,8 +15,13 @@ from ..helpers import models_tokens from ..models import CLoD, DeepSeek, OneApi, XAI -from ..utils.logging import set_verbosity_info, set_verbosity_warning +from ..utils.logging import set_verbosity_info, set_verbosity_warning, get_logger +from ..telemetry import log_graph_execution +logger = get_logger(__name__) + +# ANSI escape sequence for hyperlink +CLICKABLE_URL = "\033]8;;https://scrapegraphai.com/oss\033\\https://scrapegraphai.com/oss\033]8;;\033\\" class AbstractGraph(ABC): """ @@ -316,6 +322,10 @@ def run(self) -> str: """ Abstract method to execute the graph and return the result. """ + inputs = {"user_prompt": self.prompt, self.input_key: self.source} + self.final_state, self.execution_info = self.graph.execute(inputs) + result = self.final_state.get("answer", "No answer found.") + return result async def run_safe_async(self) -> str: """ diff --git a/scrapegraphai/graphs/base_graph.py b/scrapegraphai/graphs/base_graph.py index 6021f97f..f8489289 100644 --- a/scrapegraphai/graphs/base_graph.py +++ b/scrapegraphai/graphs/base_graph.py @@ -8,7 +8,12 @@ from ..telemetry import log_graph_execution from ..utils import CustomLLMCallbackManager +from ..utils.logging import get_logger +logger = get_logger(__name__) + +# ANSI escape sequence for hyperlink +CLICKABLE_URL = "\033]8;;https://scrapegraphai.com/oss\033\\https://scrapegraphai.com/oss\033]8;;\033\\" class BaseGraph: """ @@ -353,9 +358,24 @@ def execute(self, initial_state: dict) -> Tuple[dict, list]: bridge = BurrBridge(self, self.burr_config) result = bridge.execute(initial_state) - return (result["_state"], []) + state, exec_info = (result["_state"], []) else: - return self._execute_standard(initial_state) + state, exec_info = self._execute_standard(initial_state) + + # Print the result first + if "answer" in state: + print(state["answer"]) + elif "parsed_doc" in state: + print(state["parsed_doc"]) + elif "generated_code" in state: + print(state["generated_code"]) + elif "merged_script" in state: + print(state["merged_script"]) + + # Then show the message ONLY ONCE + print(f"✨ Try enhanced version of ScrapegraphAI at {CLICKABLE_URL} ✨") + + return state, exec_info def append_node(self, node): """ diff --git a/scrapegraphai/graphs/markdownify_graph.py b/scrapegraphai/graphs/markdownify_graph.py new file mode 100644 index 00000000..78d33b12 --- /dev/null +++ b/scrapegraphai/graphs/markdownify_graph.py @@ -0,0 +1,83 @@ +""" +markdownify_graph module +""" + +from typing import Dict, List, Optional, Tuple + +from ..nodes import ( + FetchNode, + MarkdownifyNode, +) +from .base_graph import BaseGraph + + +class MarkdownifyGraph(BaseGraph): + """ + A graph that converts HTML content to Markdown format. + + This graph takes a URL or HTML content as input and converts it to clean, readable Markdown. + It uses a two-step process: + 1. Fetch the content (if URL is provided) + 2. Convert the content to Markdown format + + Args: + llm_model: The language model to use for processing + embedder_model: The embedding model to use (optional) + node_config: Additional configuration for the nodes (optional) + + Example: + >>> graph = MarkdownifyGraph( + ... llm_model=your_llm_model, + ... embedder_model=your_embedder_model + ... ) + >>> result, _ = graph.execute({"url": "https://example.com"}) + >>> print(result["markdown"]) + """ + + def __init__( + self, + llm_model, + embedder_model=None, + node_config: Optional[Dict] = None, + ): + # Initialize nodes + fetch_node = FetchNode( + input="url | html", + output=["html_content"], + node_config=node_config, + ) + + markdownify_node = MarkdownifyNode( + input="html_content", + output=["markdown"], + node_config=node_config, + ) + + # Define graph structure + nodes = [fetch_node, markdownify_node] + edges = [(fetch_node, markdownify_node)] + + super().__init__( + nodes=nodes, + edges=edges, + entry_point=fetch_node, + graph_name="Markdownify", + ) + + def execute( + self, initial_state: Dict + ) -> Tuple[Dict, List[Dict]]: + """ + Execute the markdownify graph. + + Args: + initial_state: A dictionary containing either: + - "url": The URL to fetch and convert to markdown + - "html": The HTML content to convert to markdown + + Returns: + Tuple containing: + - Dictionary with the markdown result in the "markdown" key + - List of execution logs + """ + return super().execute(initial_state) \ No newline at end of file diff --git a/scrapegraphai/nodes/__init__.py b/scrapegraphai/nodes/__init__.py index 460e3f40..b6917238 100644 --- a/scrapegraphai/nodes/__init__.py +++ b/scrapegraphai/nodes/__init__.py @@ -20,6 +20,7 @@ from .graph_iterator_node import GraphIteratorNode from .html_analyzer_node import HtmlAnalyzerNode from .image_to_text_node import ImageToTextNode +from .markdownify_node import MarkdownifyNode from .merge_answers_node import MergeAnswersNode from .merge_generated_scripts_node import MergeGeneratedScriptsNode from .parse_node import ParseNode @@ -45,6 +46,7 @@ "ParseNode", "ParseNodeDepthK", "RobotsNode", + "MarkdownifyNode", # Analysis nodes "HtmlAnalyzerNode", "GetProbableTagsNode", diff --git a/scrapegraphai/nodes/generate_answer_node.py b/scrapegraphai/nodes/generate_answer_node.py index db4467be..e4346fe9 100644 --- a/scrapegraphai/nodes/generate_answer_node.py +++ b/scrapegraphai/nodes/generate_answer_node.py @@ -180,9 +180,8 @@ def execute(self, state: dict) -> dict: if len(doc) == 1: prompt = PromptTemplate( template=template_no_chunks_prompt, - input_variables=["question"], + input_variables=["content", "question"], partial_variables={ - "context": doc, "format_instructions": format_instructions, }, ) @@ -192,7 +191,7 @@ def execute(self, state: dict) -> dict: try: answer = self.invoke_with_timeout( - chain, {"question": user_prompt}, self.timeout + chain, {"content": doc, "question": user_prompt}, self.timeout ) except (Timeout, json.JSONDecodeError) as e: error_msg = ( @@ -216,7 +215,7 @@ def execute(self, state: dict) -> dict: template=template_chunks_prompt, input_variables=["question"], partial_variables={ - "context": chunk, + "content": chunk, "chunk_id": i + 1, "format_instructions": format_instructions, }, @@ -242,7 +241,7 @@ def execute(self, state: dict) -> dict: merge_prompt = PromptTemplate( template=template_merge_prompt, - input_variables=["context", "question"], + input_variables=["content", "question"], partial_variables={"format_instructions": format_instructions}, ) @@ -252,7 +251,7 @@ def execute(self, state: dict) -> dict: try: answer = self.invoke_with_timeout( merge_chain, - {"context": batch_results, "question": user_prompt}, + {"content": batch_results, "question": user_prompt}, self.timeout, ) except (Timeout, json.JSONDecodeError) as e: diff --git a/scrapegraphai/nodes/markdownify_node.py b/scrapegraphai/nodes/markdownify_node.py new file mode 100644 index 00000000..2119908a --- /dev/null +++ b/scrapegraphai/nodes/markdownify_node.py @@ -0,0 +1,67 @@ +""" +MarkdownifyNode Module +""" + +from typing import List, Optional + +from ..utils.convert_to_md import convert_to_md +from .base_node import BaseNode + + +class MarkdownifyNode(BaseNode): + """ + A node responsible for converting HTML content to Markdown format. + + This node takes HTML content from the state and converts it to clean, readable Markdown. + It uses the convert_to_md utility function to perform the conversion. + + Attributes: + verbose (bool): A flag indicating whether to show print statements during execution. + + Args: + input (str): Boolean expression defining the input keys needed from the state. + output (List[str]): List of output keys to be updated in the state. + node_config (Optional[dict]): Additional configuration for the node. + node_name (str): The unique identifier name for the node, defaulting to "Markdownify". + """ + + def __init__( + self, + input: str, + output: List[str], + node_config: Optional[dict] = None, + node_name: str = "Markdownify", + ): + super().__init__(node_name, "node", input, output, 1, node_config) + + self.verbose = ( + False if node_config is None else node_config.get("verbose", False) + ) + + def execute(self, state: dict) -> dict: + """ + Executes the node's logic to convert HTML content to Markdown. + + Args: + state (dict): The current state of the graph. The input keys will be used to fetch the + HTML content from the state. + + Returns: + dict: The updated state with the output key containing the Markdown content. + + Raises: + KeyError: If the input keys are not found in the state, indicating that the + necessary HTML content is missing. + """ + self.logger.info(f"--- Executing {self.node_name} Node ---") + + input_keys = self.get_input_keys(state) + html_content = state[input_keys[0]] + + # Convert HTML to Markdown + markdown_content = convert_to_md(html_content) + + # Update state with markdown content + state.update({self.output[0]: markdown_content}) + + return state \ No newline at end of file diff --git a/scrapegraphai/prompts/generate_answer_node_prompts.py b/scrapegraphai/prompts/generate_answer_node_prompts.py index 79cb3019..91e49c25 100644 --- a/scrapegraphai/prompts/generate_answer_node_prompts.py +++ b/scrapegraphai/prompts/generate_answer_node_prompts.py @@ -13,7 +13,7 @@ and things that will invalidate the dictionary. \n Do not start the response with ```json because it will invalidate the postprocessing. \n OUTPUT INSTRUCTIONS: {format_instructions}\n -Content of {chunk_id}: {context}. \n +Content of {chunk_id}: {content}. \n """ TEMPLATE_NO_CHUNKS_MD = """ @@ -27,7 +27,7 @@ Do not start the response with ```json because it will invalidate the postprocessing. \n OUTPUT INSTRUCTIONS: {format_instructions}\n USER QUESTION: {question}\n -WEBSITE CONTENT: {context}\n +WEBSITE CONTENT: {content}\n """ TEMPLATE_MERGE_MD = """ @@ -42,7 +42,7 @@ Do not start the response with ```json because it will invalidate the postprocessing. \n OUTPUT INSTRUCTIONS: {format_instructions}\n USER QUESTION: {question}\n -WEBSITE CONTENT: {context}\n +WEBSITE CONTENT: {content}\n """ TEMPLATE_CHUNKS = """ @@ -56,7 +56,7 @@ and things that will invalidate the dictionary. \n Do not start the response with ```json because it will invalidate the postprocessing. \n OUTPUT INSTRUCTIONS: {format_instructions}\n -Content of {chunk_id}: {context}. \n +Content of {chunk_id}: {content}. \n """ TEMPLATE_NO_CHUNKS = """ @@ -70,7 +70,7 @@ Do not start the response with ```json because it will invalidate the postprocessing. \n OUTPUT INSTRUCTIONS: {format_instructions}\n USER QUESTION: {question}\n -WEBSITE CONTENT: {context}\n +WEBSITE CONTENT: {content}\n """ TEMPLATE_MERGE = """ @@ -84,7 +84,7 @@ Do not start the response with ```json because it will invalidate the postprocessing. \n OUTPUT INSTRUCTIONS: {format_instructions}\n USER QUESTION: {question}\n -WEBSITE CONTENT: {context}\n +WEBSITE CONTENT: {content}\n """ REGEN_ADDITIONAL_INFO = """ diff --git a/uv.lock b/uv.lock index 9ab0fd3e..d2c7bfc5 100644 --- a/uv.lock +++ b/uv.lock @@ -1,23 +1,51 @@ version = 1 -revision = 1 requires-python = ">=3.10, <4.0" resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'darwin'", - "python_version < '0'", - "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", - "python_full_version == '3.11.*' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux')", - "python_full_version >= '3.12' and python_full_version < '3.12.4' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and sys_platform != 'darwin' and sys_platform != 'linux')", - "python_full_version >= '3.12.4' and python_full_version < '3.13' and sys_platform == 'darwin'", - "python_full_version >= '3.13' and sys_platform == 'darwin'", - "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12.4' and python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')", - "(python_full_version >= '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.11' and platform_system == 'Darwin' and sys_platform == 'darwin'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform == 'darwin') or (python_full_version < '3.11' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'darwin')", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_system == 'Darwin' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_system == 'Darwin' and sys_platform != 'darwin') or (python_full_version < '3.11' and platform_system == 'Darwin' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform != 'darwin') or (python_full_version < '3.11' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.11.*' and platform_system == 'Darwin' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform == 'darwin') or (python_full_version == '3.11.*' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'darwin')", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_system == 'Darwin' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_system == 'Darwin' and sys_platform != 'darwin') or (python_full_version == '3.11.*' and platform_system == 'Darwin' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform != 'darwin') or (python_full_version == '3.11.*' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin' and sys_platform == 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'darwin'", + "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform == 'darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'darwin')", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Darwin' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system == 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux'", + "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_system == 'Darwin' and sys_platform == 'darwin'", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'darwin'", + "(python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform == 'darwin') or (python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'darwin')", + "python_full_version >= '3.13' and platform_system == 'Darwin' and sys_platform == 'darwin'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'darwin'", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform == 'darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'darwin')", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Darwin' and sys_platform == 'linux'", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'linux'", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'linux'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Darwin' and sys_platform == 'linux'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'linux'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'linux'", + "(python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine != 'aarch64' and platform_system == 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_system == 'Darwin' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux'", + "(python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux')", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system == 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.13' and platform_system == 'Darwin' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux'", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux')", ] [[package]] @@ -206,12 +234,24 @@ name = "async-timeout" version = "4.0.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform == 'darwin'", - "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux')", - "python_full_version == '3.11.*' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.11' and platform_system == 'Darwin' and sys_platform == 'darwin'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'darwin'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform == 'darwin') or (python_full_version < '3.11' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'darwin')", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_system == 'Darwin' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_system == 'Darwin' and sys_platform != 'darwin') or (python_full_version < '3.11' and platform_system == 'Darwin' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.11' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform != 'darwin') or (python_full_version < '3.11' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.11.*' and platform_system == 'Darwin' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'darwin'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform == 'darwin') or (python_full_version == '3.11.*' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'darwin')", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_system == 'Darwin' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_system == 'Darwin' and sys_platform != 'darwin') or (python_full_version == '3.11.*' and platform_system == 'Darwin' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform != 'darwin') or (python_full_version == '3.11.*' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux')", ] sdist = { url = "https://files.pythonhosted.org/packages/87/d6/21b30a550dafea84b1b8eee21b5e23fa16d010ae006011221f33dcd8d7f8/async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f", size = 8345 } wheels = [ @@ -223,15 +263,33 @@ name = "async-timeout" version = "5.0.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12' and python_full_version < '3.12.4' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and sys_platform != 'darwin' and sys_platform != 'linux')", - "python_full_version >= '3.12.4' and python_full_version < '3.13' and sys_platform == 'darwin'", - "python_full_version >= '3.13' and sys_platform == 'darwin'", - "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.12.4' and python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')", - "(python_full_version >= '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin' and sys_platform == 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'darwin'", + "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform == 'darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'darwin')", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Darwin' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system == 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system == 'Darwin' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux'", + "(python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.12' and python_full_version < '3.12.4' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_system == 'Darwin' and sys_platform == 'darwin'", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'darwin'", + "(python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform == 'darwin') or (python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'darwin')", + "python_full_version >= '3.13' and platform_system == 'Darwin' and sys_platform == 'darwin'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'darwin'", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform == 'darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'darwin')", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Darwin' and sys_platform == 'linux'", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'linux'", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'linux'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Darwin' and sys_platform == 'linux'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform == 'linux'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform == 'linux'", + "(python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine != 'aarch64' and platform_system == 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_system == 'Darwin' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux'", + "(python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.12.4' and python_full_version < '3.13' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux')", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system == 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.13' and platform_system == 'Darwin' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_system == 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux'", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin' and sys_platform != 'darwin') or (python_full_version >= '3.13' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'darwin' and sys_platform != 'linux')", ] sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274 } wheels = [ @@ -467,7 +525,7 @@ name = "click" version = "8.1.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "platform_system == 'Windows'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } wheels = [ @@ -1853,7 +1911,7 @@ version = "2.10.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pygments" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "platform_system == 'Windows'" }, { name = "tqdm" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3a/93/80ac75c20ce54c785648b4ed363c88f148bf22637e10c9863db4fbe73e74/mpire-2.10.2.tar.gz", hash = "sha256:f66a321e93fadff34585a4bfa05e95bd946cf714b442f51c529038eb45773d97", size = 271270 } @@ -2078,6 +2136,7 @@ name = "nvidia-cublas-cu12" version = "12.4.5.8" source = { registry = "https://pypi.org/simple" } wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/7f/7fbae15a3982dc9595e49ce0f19332423b260045d0a6afe93cdbe2f1f624/nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0f8aa1706812e00b9f19dfe0cdb3999b092ccb8ca168c0db5b8ea712456fd9b3", size = 363333771 }, { url = "https://files.pythonhosted.org/packages/ae/71/1c91302526c45ab494c23f61c7a84aa568b8c1f9d196efa5993957faf906/nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:2fc8da60df463fdefa81e323eef2e36489e1c94335b5358bcb38360adf75ac9b", size = 363438805 }, ] @@ -2086,6 +2145,7 @@ name = "nvidia-cuda-cupti-cu12" version = "12.4.127" source = { registry = "https://pypi.org/simple" } wheels = [ + { url = "https://files.pythonhosted.org/packages/93/b5/9fb3d00386d3361b03874246190dfec7b206fd74e6e287b26a8fcb359d95/nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:79279b35cf6f91da114182a5ce1864997fd52294a87a16179ce275773799458a", size = 12354556 }, { url = "https://files.pythonhosted.org/packages/67/42/f4f60238e8194a3106d06a058d494b18e006c10bb2b915655bd9f6ea4cb1/nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9dec60f5ac126f7bb551c055072b69d85392b13311fcc1bcda2202d172df30fb", size = 13813957 }, ] @@ -2094,6 +2154,7 @@ name = "nvidia-cuda-nvrtc-cu12" version = "12.4.127" source = { registry = "https://pypi.org/simple" } wheels = [ + { url = "https://files.pythonhosted.org/packages/77/aa/083b01c427e963ad0b314040565ea396f914349914c298556484f799e61b/nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0eedf14185e04b76aa05b1fea04133e59f465b6f960c0cbf4e37c3cb6b0ea198", size = 24133372 }, { url = "https://files.pythonhosted.org/packages/2c/14/91ae57cd4db3f9ef7aa99f4019cfa8d54cb4caa7e00975df6467e9725a9f/nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a178759ebb095827bd30ef56598ec182b85547f1508941a3d560eb7ea1fbf338", size = 24640306 }, ] @@ -2102,6 +2163,7 @@ name = "nvidia-cuda-runtime-cu12" version = "12.4.127" source = { registry = "https://pypi.org/simple" } wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/aa/b656d755f474e2084971e9a297def515938d56b466ab39624012070cb773/nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:961fe0e2e716a2a1d967aab7caee97512f71767f852f67432d572e36cb3a11f3", size = 894177 }, { url = "https://files.pythonhosted.org/packages/ea/27/1795d86fe88ef397885f2e580ac37628ed058a92ed2c39dc8eac3adf0619/nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:64403288fa2136ee8e467cdc9c9427e0434110899d07c779f25b5c068934faa5", size = 883737 }, ] @@ -2110,7 +2172,7 @@ name = "nvidia-cudnn-cu12" version = "9.1.0.70" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/9f/fd/713452cd72343f682b1c7b9321e23829f00b842ceaedcda96e742ea0b0b3/nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f", size = 664752741 }, @@ -2121,9 +2183,10 @@ name = "nvidia-cufft-cu12" version = "11.2.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'linux')" }, ] wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/8a/0e728f749baca3fbeffad762738276e5df60851958be7783af121a7221e7/nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5dad8008fc7f92f5ddfa2101430917ce2ffacd86824914c82e28990ad7f00399", size = 211422548 }, { url = "https://files.pythonhosted.org/packages/27/94/3266821f65b92b3138631e9c8e7fe1fb513804ac934485a8d05776e1dd43/nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9", size = 211459117 }, ] @@ -2132,6 +2195,7 @@ name = "nvidia-curand-cu12" version = "10.3.5.147" source = { registry = "https://pypi.org/simple" } wheels = [ + { url = "https://files.pythonhosted.org/packages/80/9c/a79180e4d70995fdf030c6946991d0171555c6edf95c265c6b2bf7011112/nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1f173f09e3e3c76ab084aba0de819c49e56614feae5c12f69883f4ae9bb5fad9", size = 56314811 }, { url = "https://files.pythonhosted.org/packages/8a/6d/44ad094874c6f1b9c654f8ed939590bdc408349f137f9b98a3a23ccec411/nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a88f583d4e0bb643c49743469964103aa59f7f708d862c3ddb0fc07f851e3b8b", size = 56305206 }, ] @@ -2140,11 +2204,12 @@ name = "nvidia-cusolver-cu12" version = "11.6.1.9" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, - { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, - { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'linux')" }, + { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'linux')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'linux')" }, ] wheels = [ + { url = "https://files.pythonhosted.org/packages/46/6b/a5c33cf16af09166845345275c34ad2190944bcc6026797a39f8e0a282e0/nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d338f155f174f90724bbde3758b7ac375a70ce8e706d70b018dd3375545fc84e", size = 127634111 }, { url = "https://files.pythonhosted.org/packages/3a/e1/5b9089a4b2a4790dfdea8b3a006052cfecff58139d5a4e34cb1a51df8d6f/nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260", size = 127936057 }, ] @@ -2153,9 +2218,10 @@ name = "nvidia-cusparse-cu12" version = "12.3.1.170" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'linux')" }, ] wheels = [ + { url = "https://files.pythonhosted.org/packages/96/a9/c0d2f83a53d40a4a41be14cea6a0bf9e668ffcf8b004bd65633f433050c0/nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_aarch64.whl", hash = "sha256:9d32f62896231ebe0480efd8a7f702e143c98cfaa0e8a76df3386c1ba2b54df3", size = 207381987 }, { url = "https://files.pythonhosted.org/packages/db/f7/97a9ea26ed4bbbfc2d470994b8b4f338ef663be97b8f677519ac195e113d/nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1", size = 207454763 }, ] @@ -2172,6 +2238,7 @@ name = "nvidia-nvjitlink-cu12" version = "12.4.127" source = { registry = "https://pypi.org/simple" } wheels = [ + { url = "https://files.pythonhosted.org/packages/02/45/239d52c05074898a80a900f49b1615d81c07fceadd5ad6c4f86a987c0bc4/nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4abe7fef64914ccfa909bc2ba39739670ecc9e820c83ccc7a6ed414122599b83", size = 20552510 }, { url = "https://files.pythonhosted.org/packages/ff/ff/847841bacfbefc97a00036e0fce5a0f086b640756dc38caea5e1bb002655/nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57", size = 21066810 }, ] @@ -2180,6 +2247,7 @@ name = "nvidia-nvtx-cu12" version = "12.4.127" source = { registry = "https://pypi.org/simple" } wheels = [ + { url = "https://files.pythonhosted.org/packages/06/39/471f581edbb7804b39e8063d92fc8305bdc7a80ae5c07dbe6ea5c50d14a5/nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7959ad635db13edf4fc65c06a6e9f9e55fc2f92596db928d169c0bb031e88ef3", size = 100417 }, { url = "https://files.pythonhosted.org/packages/87/20/199b8713428322a2f22b722c62b8cc278cc53dffa9705d744484b5035ee9/nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a", size = 99144 }, ] @@ -3376,9 +3444,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/46/5d11dc300feaad285c2f1bd784ff3f689f5e0ab6be49aaf568f3a77019eb/safetensors-0.4.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:21742b391b859e67b26c0b2ac37f52c9c0944a879a25ad2f9f9f3cd61e7fda8f", size = 606660 }, ] +[[package]] +name = "scrapegraph-py" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "beautifulsoup4" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/36/10546d18157cd2efb2de09098fac8dd6f689b98842a4cc71bb7fc29ba4b9/scrapegraph_py-1.12.0.tar.gz", hash = "sha256:82d27e8ea325975f768f80d4edf403b6294518dae6a1e3ae63e27b8934a5dacb", size = 113290 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/a8/8610143e9ebad9596e402260f63cbb6168f99719f07e13847b1df5a28f4d/scrapegraph_py-1.12.0-py3-none-any.whl", hash = "sha256:fd74d091529d3f8f5ba057950333e15a48ac5c0be7e2a56a8f2bad04cebdac30", size = 15458 }, +] + [[package]] name = "scrapegraphai" -version = "1.43.0" +version = "1.56.0" source = { editable = "." } dependencies = [ { name = "async-timeout", version = "4.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, @@ -3398,6 +3482,7 @@ dependencies = [ { name = "playwright" }, { name = "pydantic" }, { name = "python-dotenv" }, + { name = "scrapegraph-py" }, { name = "semchunk" }, { name = "simpleeval" }, { name = "tiktoken" }, @@ -3460,6 +3545,7 @@ requires-dist = [ { name = "playwright", specifier = ">=1.43.0" }, { name = "pydantic", specifier = ">=2.10.2" }, { name = "python-dotenv", specifier = ">=1.0.1" }, + { name = "scrapegraph-py", specifier = ">=0.1.0" }, { name = "semchunk", specifier = ">=2.2.0" }, { name = "simpleeval", specifier = ">=1.0.0" }, { name = "sphinx", marker = "extra == 'docs'", specifier = "==6.0" }, @@ -3468,7 +3554,6 @@ requires-dist = [ { name = "tqdm", specifier = ">=4.66.4" }, { name = "undetected-playwright", specifier = ">=0.3.0" }, ] -provides-extras = ["burr", "docs", "ocr"] [package.metadata.requires-dev] dev = [ @@ -3757,7 +3842,7 @@ dependencies = [ { name = "toml" }, { name = "tornado" }, { name = "typing-extensions" }, - { name = "watchdog", marker = "sys_platform != 'darwin'" }, + { name = "watchdog", marker = "platform_system != 'Darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b0/e5/2bf2daa9c98658f1474bb64e7de030cbc4182b5f2b2196536efedaef02cb/streamlit-1.40.2.tar.gz", hash = "sha256:0cc131fc9b18065feaff8f6f241c81164ad37d8d9e3a85499a0240aaaf6a6a61", size = 8265763 } wheels = [ @@ -3961,21 +4046,21 @@ dependencies = [ { name = "fsspec" }, { name = "jinja2" }, { name = "networkx" }, - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, { name = "setuptools", marker = "python_full_version >= '3.12'" }, { name = "sympy" }, - { name = "triton", marker = "python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "triton", marker = "python_full_version < '3.13' and platform_machine == 'x86_64' and platform_system == 'Linux'" }, { name = "typing-extensions" }, ] wheels = [ @@ -4017,7 +4102,7 @@ name = "tqdm" version = "4.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "platform_system == 'Windows'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } wheels = [ @@ -4059,7 +4144,7 @@ name = "triton" version = "3.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock", marker = "(python_full_version < '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "filelock", marker = "(python_full_version < '3.13' and platform_machine != 'aarch64' and platform_system != 'Darwin') or (python_full_version < '3.13' and platform_system != 'Darwin' and platform_system != 'Linux' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/98/29/69aa56dc0b2eb2602b553881e34243475ea2afd9699be042316842788ff5/triton-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b0dd10a925263abbe9fa37dcde67a5e9b2383fc269fdf59f5657cac38c5d1d8", size = 209460013 },