Skip to content

Commit c6001b8

Browse files
PeriniMPeriniM
committed
chore(tests): updated tests
1 parent 11d172b commit c6001b8

File tree

3 files changed

+183
-102
lines changed

3 files changed

+183
-102
lines changed

‎scrapegraph-py/tests/test_async_client.py‎

Lines changed: 64 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ def mock_uuid():
1919

2020

2121
@pytest.mark.asyncio
22-
async def test_smartscraper(mock_api_key):
22+
async def test_smartscraper_with_url(mock_api_key):
2323
with aioresponses() as mocked:
2424
mocked.post(
2525
"https://api.scrapegraphai.com/v1/smartscraper",
@@ -38,6 +38,54 @@ async def test_smartscraper(mock_api_key):
3838
assert "description" in response["result"]
3939

4040

41+
@pytest.mark.asyncio
42+
async def test_smartscraper_with_html(mock_api_key):
43+
with aioresponses() as mocked:
44+
mocked.post(
45+
"https://api.scrapegraphai.com/v1/smartscraper",
46+
payload={
47+
"request_id": str(uuid4()),
48+
"status": "completed",
49+
"result": {"description": "Test content."},
50+
},
51+
)
52+
53+
async with AsyncClient(api_key=mock_api_key) as client:
54+
response = await client.smartscraper(
55+
website_html="<html><body><p>Test content</p></body></html>",
56+
user_prompt="Extract info",
57+
)
58+
assert response["status"] == "completed"
59+
assert "description" in response["result"]
60+
61+
62+
@pytest.mark.asyncio
63+
async def test_smartscraper_with_headers(mock_api_key):
64+
with aioresponses() as mocked:
65+
mocked.post(
66+
"https://api.scrapegraphai.com/v1/smartscraper",
67+
payload={
68+
"request_id": str(uuid4()),
69+
"status": "completed",
70+
"result": {"description": "Example domain."},
71+
},
72+
)
73+
74+
headers = {
75+
"User-Agent": "Mozilla/5.0",
76+
"Cookie": "session=123",
77+
}
78+
79+
async with AsyncClient(api_key=mock_api_key) as client:
80+
response = await client.smartscraper(
81+
website_url="https://example.com",
82+
user_prompt="Describe this page.",
83+
headers=headers,
84+
)
85+
assert response["status"] == "completed"
86+
assert "description" in response["result"]
87+
88+
4189
@pytest.mark.asyncio
4290
async def test_get_credits(mock_api_key):
4391
with aioresponses() as mocked:
@@ -122,57 +170,43 @@ async def test_markdownify(mock_api_key):
122170

123171

124172
@pytest.mark.asyncio
125-
async def test_get_markdownify(mock_api_key, mock_uuid):
126-
with aioresponses() as mocked:
127-
mocked.get(
128-
f"https://api.scrapegraphai.com/v1/markdownify/{mock_uuid}",
129-
payload={
130-
"request_id": mock_uuid,
131-
"status": "completed",
132-
"result": "# Example Page\n\nThis is markdown content.",
133-
},
134-
)
135-
136-
async with AsyncClient(api_key=mock_api_key) as client:
137-
response = await client.get_markdownify(mock_uuid)
138-
assert response["status"] == "completed"
139-
assert response["request_id"] == mock_uuid
140-
141-
142-
@pytest.mark.asyncio
143-
async def test_localscraper(mock_api_key):
173+
async def test_markdownify_with_headers(mock_api_key):
144174
with aioresponses() as mocked:
145175
mocked.post(
146-
"https://api.scrapegraphai.com/v1/localscraper",
176+
"https://api.scrapegraphai.com/v1/markdownify",
147177
payload={
148178
"request_id": str(uuid4()),
149179
"status": "completed",
150-
"result": {"extracted_info": "Test content"},
180+
"result": "# Example Page\n\nThis is markdown content.",
151181
},
152182
)
153183

184+
headers = {
185+
"User-Agent": "Mozilla/5.0",
186+
"Cookie": "session=123",
187+
}
188+
154189
async with AsyncClient(api_key=mock_api_key) as client:
155-
response = await client.localscraper(
156-
user_prompt="Extract info",
157-
website_html="<html><body><p>Test content</p></body></html>",
190+
response = await client.markdownify(
191+
website_url="https://example.com", headers=headers
158192
)
159193
assert response["status"] == "completed"
160-
assert "extracted_info" in response["result"]
194+
assert "# Example Page" in response["result"]
161195

162196

163197
@pytest.mark.asyncio
164-
async def test_get_localscraper(mock_api_key, mock_uuid):
198+
async def test_get_markdownify(mock_api_key, mock_uuid):
165199
with aioresponses() as mocked:
166200
mocked.get(
167-
f"https://api.scrapegraphai.com/v1/localscraper/{mock_uuid}",
201+
f"https://api.scrapegraphai.com/v1/markdownify/{mock_uuid}",
168202
payload={
169203
"request_id": mock_uuid,
170204
"status": "completed",
171-
"result": {"extracted_info": "Test content"},
205+
"result": "# Example Page\n\nThis is markdown content.",
172206
},
173207
)
174208

175209
async with AsyncClient(api_key=mock_api_key) as client:
176-
response = await client.get_localscraper(mock_uuid)
210+
response = await client.get_markdownify(mock_uuid)
177211
assert response["status"] == "completed"
178212
assert response["request_id"] == mock_uuid

‎scrapegraph-py/tests/test_client.py‎

Lines changed: 64 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ def mock_uuid():
1818

1919

2020
@responses.activate
21-
def test_smartscraper(mock_api_key):
21+
def test_smartscraper_with_url(mock_api_key):
2222
# Mock the API response
2323
responses.add(
2424
responses.POST,
@@ -37,6 +37,54 @@ def test_smartscraper(mock_api_key):
3737
assert response["status"] == "completed"
3838

3939

40+
@responses.activate
41+
def test_smartscraper_with_html(mock_api_key):
42+
# Mock the API response
43+
responses.add(
44+
responses.POST,
45+
"https://api.scrapegraphai.com/v1/smartscraper",
46+
json={
47+
"request_id": str(uuid4()),
48+
"status": "completed",
49+
"result": {"description": "Test content."},
50+
},
51+
)
52+
53+
with Client(api_key=mock_api_key) as client:
54+
response = client.smartscraper(
55+
website_html="<html><body><p>Test content</p></body></html>",
56+
user_prompt="Extract info",
57+
)
58+
assert response["status"] == "completed"
59+
60+
61+
@responses.activate
62+
def test_smartscraper_with_headers(mock_api_key):
63+
# Mock the API response
64+
responses.add(
65+
responses.POST,
66+
"https://api.scrapegraphai.com/v1/smartscraper",
67+
json={
68+
"request_id": str(uuid4()),
69+
"status": "completed",
70+
"result": {"description": "Example domain."},
71+
},
72+
)
73+
74+
headers = {
75+
"User-Agent": "Mozilla/5.0",
76+
"Cookie": "session=123",
77+
}
78+
79+
with Client(api_key=mock_api_key) as client:
80+
response = client.smartscraper(
81+
website_url="https://example.com",
82+
user_prompt="Describe this page.",
83+
headers=headers,
84+
)
85+
assert response["status"] == "completed"
86+
87+
4088
@responses.activate
4189
def test_get_smartscraper(mock_api_key, mock_uuid):
4290
responses.add(
@@ -118,57 +166,43 @@ def test_markdownify(mock_api_key):
118166

119167

120168
@responses.activate
121-
def test_get_markdownify(mock_api_key, mock_uuid):
122-
responses.add(
123-
responses.GET,
124-
f"https://api.scrapegraphai.com/v1/markdownify/{mock_uuid}",
125-
json={
126-
"request_id": mock_uuid,
127-
"status": "completed",
128-
"result": "# Example Page\n\nThis is markdown content.",
129-
},
130-
)
131-
132-
with Client(api_key=mock_api_key) as client:
133-
response = client.get_markdownify(mock_uuid)
134-
assert response["status"] == "completed"
135-
assert response["request_id"] == mock_uuid
136-
137-
138-
@responses.activate
139-
def test_localscraper(mock_api_key):
169+
def test_markdownify_with_headers(mock_api_key):
140170
responses.add(
141171
responses.POST,
142-
"https://api.scrapegraphai.com/v1/localscraper",
172+
"https://api.scrapegraphai.com/v1/markdownify",
143173
json={
144174
"request_id": str(uuid4()),
145175
"status": "completed",
146-
"result": {"extracted_info": "Test content"},
176+
"result": "# Example Page\n\nThis is markdown content.",
147177
},
148178
)
149179

180+
headers = {
181+
"User-Agent": "Mozilla/5.0",
182+
"Cookie": "session=123",
183+
}
184+
150185
with Client(api_key=mock_api_key) as client:
151-
response = client.localscraper(
152-
user_prompt="Extract info",
153-
website_html="<html><body><p>Test content</p></body></html>",
186+
response = client.markdownify(
187+
website_url="https://example.com", headers=headers
154188
)
155189
assert response["status"] == "completed"
156-
assert "extracted_info" in response["result"]
190+
assert "# Example Page" in response["result"]
157191

158192

159193
@responses.activate
160-
def test_get_localscraper(mock_api_key, mock_uuid):
194+
def test_get_markdownify(mock_api_key, mock_uuid):
161195
responses.add(
162196
responses.GET,
163-
f"https://api.scrapegraphai.com/v1/localscraper/{mock_uuid}",
197+
f"https://api.scrapegraphai.com/v1/markdownify/{mock_uuid}",
164198
json={
165199
"request_id": mock_uuid,
166200
"status": "completed",
167-
"result": {"extracted_info": "Test content"},
201+
"result": "# Example Page\n\nThis is markdown content.",
168202
},
169203
)
170204

171205
with Client(api_key=mock_api_key) as client:
172-
response = client.get_localscraper(mock_uuid)
206+
response = client.get_markdownify(mock_uuid)
173207
assert response["status"] == "completed"
174208
assert response["request_id"] == mock_uuid

0 commit comments

Comments
 (0)