Skip to content

Commit 2010ea1

Browse files
VinciGit00VinciGit00
committed
feat: add file for python
1 parent b059fd6 commit 2010ea1

File tree

9 files changed

+639
-0
lines changed

9 files changed

+639
-0
lines changed

‎scrapegraph-py/examples/async/async_agenticscraper_example.py

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import asyncio
2+
import os
3+
4+
from dotenv import load_dotenv
5+
6+
from scrapegraph_py import AsyncClient
7+
from scrapegraph_py.logger import sgai_logger
8+
9+
# Load environment variables from .env file
10+
load_dotenv()
11+
12+
sgai_logger.set_logging(level="INFO")
13+
14+
15+
async def main():
16+
# Initialize async client with API key from environment variable
17+
api_key = os.getenv("SGAI_API_KEY")
18+
if not api_key:
19+
print("❌ Error: SGAI_API_KEY environment variable not set")
20+
print("Please either:")
21+
print(" 1. Set environment variable: export SGAI_API_KEY='your-api-key-here'")
22+
print(" 2. Create a .env file with: SGAI_API_KEY=your-api-key-here")
23+
return
24+
25+
sgai_client = AsyncClient(api_key=api_key)
26+
27+
# AgenticScraper request - automated login example
28+
response = await sgai_client.agenticscraper(
29+
url="https://dashboard.scrapegraphai.com/",
30+
use_session=True,
31+
steps=[
32+
"Type email@gmail.com in email input box",
33+
"Type test-password@123 in password inputbox",
34+
"click on login"
35+
]
36+
)
37+
38+
# Print the response
39+
print(f"Request ID: {response['request_id']}")
40+
print(f"Result: {response['result']}")
41+
42+
await sgai_client.close()
43+
44+
45+
if __name__ == "__main__":
46+
asyncio.run(main())

‎scrapegraph-py/examples/async/steps/async_step_by_step_agenticscraper_example.py

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Async Step-by-Step AgenticScraper Example
4+
5+
This example demonstrates how to use the AgenticScraper API asynchronously
6+
for automated browser interactions with proper async/await patterns.
7+
"""
8+
9+
import asyncio
10+
import json
11+
import os
12+
import time
13+
14+
import aiohttp
15+
from dotenv import load_dotenv
16+
17+
# Load environment variables from .env file
18+
load_dotenv()
19+
20+
21+
async def agentic_scraper_request():
22+
"""Example of making an async request to the agentic scraper API"""
23+
24+
# Get API key from .env file
25+
api_key = os.getenv("SGAI_API_KEY")
26+
if not api_key:
27+
raise ValueError(
28+
"API key must be provided or set in .env file as SGAI_API_KEY. "
29+
"Create a .env file with: SGAI_API_KEY=your_api_key_here"
30+
)
31+
32+
steps = [
33+
"Type email@gmail.com in email input box",
34+
"Type test-password@123 in password inputbox",
35+
"click on login"
36+
]
37+
website_url = "https://dashboard.scrapegraphai.com/"
38+
39+
headers = {
40+
"SGAI-APIKEY": api_key,
41+
"Content-Type": "application/json",
42+
}
43+
44+
body = {
45+
"url": website_url,
46+
"use_session": True,
47+
"steps": steps,
48+
}
49+
50+
print("🤖 Starting Async Agentic Scraper with Automated Actions...")
51+
print(f"🌐 Website URL: {website_url}")
52+
print(f"🔧 Use Session: True")
53+
print(f"📋 Steps: {len(steps)} automated actions")
54+
print("\n" + "=" * 60)
55+
56+
# Start timer
57+
start_time = time.time()
58+
print(
59+
f"⏱️ Timer started at: {time.strftime('%H:%M:%S', time.localtime(start_time))}"
60+
)
61+
print("🔄 Processing request asynchronously...")
62+
63+
try:
64+
async with aiohttp.ClientSession() as session:
65+
async with session.post(
66+
"http://localhost:8001/v1/agentic-scrapper",
67+
json=body,
68+
headers=headers,
69+
) as response:
70+
# Calculate execution time
71+
end_time = time.time()
72+
execution_time = end_time - start_time
73+
execution_minutes = execution_time / 60
74+
75+
print(
76+
f"⏱️ Timer stopped at: {time.strftime('%H:%M:%S', time.localtime(end_time))}"
77+
)
78+
print(
79+
f"⚡ Total execution time: {execution_time:.2f} seconds ({execution_minutes:.2f} minutes)"
80+
)
81+
print(
82+
f"📊 Performance: {execution_time:.1f}s ({execution_minutes:.1f}m) for {len(steps)} steps"
83+
)
84+
85+
if response.status == 200:
86+
result = await response.json()
87+
print("✅ Request completed successfully!")
88+
print(f"📊 Request ID: {result.get('request_id', 'N/A')}")
89+
print(f"🔄 Status: {result.get('status', 'N/A')}")
90+
91+
if result.get("error"):
92+
print(f"❌ Error: {result['error']}")
93+
else:
94+
print("\n📋 EXTRACTED DATA:")
95+
print("=" * 60)
96+
97+
# Pretty print the result with proper indentation
98+
if "result" in result:
99+
print(json.dumps(result["result"], indent=2, ensure_ascii=False))
100+
else:
101+
print("No result data found")
102+
103+
else:
104+
response_text = await response.text()
105+
print(f"❌ Request failed with status code: {response.status}")
106+
print(f"Response: {response_text}")
107+
108+
except aiohttp.ClientError as e:
109+
end_time = time.time()
110+
execution_time = end_time - start_time
111+
execution_minutes = execution_time / 60
112+
print(
113+
f"⏱️ Timer stopped at: {time.strftime('%H:%M:%S', time.localtime(end_time))}"
114+
)
115+
print(
116+
f"⚡ Execution time before error: {execution_time:.2f} seconds ({execution_minutes:.2f} minutes)"
117+
)
118+
print(f"🌐 Network error: {str(e)}")
119+
except Exception as e:
120+
end_time = time.time()
121+
execution_time = end_time - start_time
122+
execution_minutes = execution_time / 60
123+
print(
124+
f"⏱️ Timer stopped at: {time.strftime('%H:%M:%S', time.localtime(end_time))}"
125+
)
126+
print(
127+
f"⚡ Execution time before error: {execution_time:.2f} seconds ({execution_minutes:.2f} minutes)"
128+
)
129+
print(f"💥 Unexpected error: {str(e)}")
130+
131+
132+
def show_curl_equivalent():
133+
"""Show the equivalent curl command for reference"""
134+
135+
# Load environment variables from .env file
136+
load_dotenv()
137+
138+
api_key = os.getenv("SGAI_API_KEY", "your-api-key-here")
139+
curl_command = f"""
140+
curl --location 'http://localhost:8001/v1/agentic-scrapper' \\
141+
--header 'SGAI-APIKEY: {api_key}' \\
142+
--header 'Content-Type: application/json' \\
143+
--data-raw '{{
144+
"url": "https://dashboard.scrapegraphai.com/",
145+
"use_session": true,
146+
"steps": [
147+
"Type email@gmail.com in email input box",
148+
"Type test-password@123 in password inputbox",
149+
"click on login"
150+
]
151+
}}'
152+
"""
153+
154+
print("Equivalent curl command:")
155+
print(curl_command)
156+
157+
158+
async def main():
159+
"""Main async function to run the agentic scraper example"""
160+
try:
161+
print("🤖 ASYNC AGENTIC SCRAPER EXAMPLE")
162+
print("=" * 60)
163+
print("This example demonstrates async automated browser interactions")
164+
print()
165+
166+
# Show the curl equivalent
167+
show_curl_equivalent()
168+
169+
print("\n" + "=" * 60)
170+
171+
# Make the actual API request
172+
await agentic_scraper_request()
173+
174+
print("\n" + "=" * 60)
175+
print("Example completed!")
176+
print("\nKey takeaways:")
177+
print("1. Async agentic scraper enables non-blocking automation")
178+
print("2. Each step is executed sequentially but asynchronously")
179+
print("3. Session management allows for complex workflows")
180+
print("4. Perfect for concurrent automation tasks")
181+
print("\nNext steps:")
182+
print("- Run multiple agentic scrapers concurrently")
183+
print("- Combine with other async operations")
184+
print("- Implement async error handling")
185+
print("- Use async session management for efficiency")
186+
187+
except Exception as e:
188+
print(f"💥 Error occurred: {str(e)}")
189+
print("\n🛠️ Troubleshooting:")
190+
print("1. Make sure your .env file contains SGAI_API_KEY")
191+
print("2. Ensure the API server is running on localhost:8001")
192+
print("3. Check your internet connection")
193+
print("4. Verify the target website is accessible")
194+
195+
196+
if __name__ == "__main__":
197+
asyncio.run(main())

‎scrapegraph-py/examples/sync/agenticscraper_example.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import os
2+
3+
from dotenv import load_dotenv
4+
5+
from scrapegraph_py import Client
6+
from scrapegraph_py.logger import sgai_logger
7+
8+
# Load environment variables from .env file
9+
load_dotenv()
10+
11+
sgai_logger.set_logging(level="INFO")
12+
13+
# Initialize the client with API key from environment variable
14+
api_key = os.getenv("SGAI_API_KEY")
15+
if not api_key:
16+
print("❌ Error: SGAI_API_KEY environment variable not set")
17+
print("Please either:")
18+
print(" 1. Set environment variable: export SGAI_API_KEY='your-api-key-here'")
19+
print(" 2. Create a .env file with: SGAI_API_KEY=your-api-key-here")
20+
exit(1)
21+
22+
sgai_client = Client(api_key=api_key)
23+
24+
# AgenticScraper request - automated login example
25+
response = sgai_client.agenticscraper(
26+
url="https://dashboard.scrapegraphai.com/",
27+
use_session=True,
28+
steps=[
29+
"Type email@gmail.com in email input box",
30+
"Type test-password@123 in password inputbox",
31+
"click on login"
32+
]
33+
)
34+
35+
# Print the response
36+
print(f"Request ID: {response['request_id']}")
37+
print(f"Result: {response['result']}")
38+
39+
sgai_client.close()

0 commit comments

Comments
 (0)