Loading...
Loading...
Web scraping using the Scrapfly Scraper API with the Python SDK
npx skill4agent add scrapfly/skills scrapfly-scraperpip install scrapfly-sdkSCRAPFLY_API_KEYhttps://api.scrapfly.io/scrapemethodScrapeConfigfrom scrapfly import ScrapflyClient, ScrapeConfig, ScrapeApiResponse
import os
client = ScrapflyClient(key=os.environ["SCRAPFLY_API_KEY"])| Parameter | Type | Default | Description |
|---|---|---|---|
| str | required | Target URL to scrape |
| str | | HTTP method: GET, POST, PUT, PATCH, HEAD, OPTIONS |
| dict | None | Custom HTTP headers |
| dict | None | Custom cookies (merged into |
| str | None | Raw request body for POST/PUT/PATCH; used when you pre-encode payload |
| dict | None | Form/JSON data for POST/PUT/PATCH; encoded according to |
| int | None | Request timeout in ms (default ~150,000ms) |
| bool | True | Auto-retry on network failures |
| str | None | Proxy country (ISO 3166-1 alpha-2, e.g. |
| str | | Proxy pool: |
| str | None | Session ID to persist cookies/fingerprint across requests |
| bool | False | Keep the same proxy IP for a given |
| bool | False | Enable the Unblocker anti-bot bypass. |
| bool | False | Enable headless browser JavaScript rendering (+5 credits) |
| int | None | Wait time in ms after page load (requires |
| str | | Browser readiness stage: |
| str | None | CSS/XPath selector to wait for (requires |
| str | None | JavaScript code to execute in the browser (auto-encoded) |
| list | None | List of browser actions (click, fill, scroll, wait, etc.) |
| bool | None | Automatically scroll page during rendering for lazy-loaded content (requires |
| dict | None | Capture screenshots: |
| list[str] | None | Screenshot options: |
| str | | Output format: |
| list[str] | None | Format modifiers (markdown only): |
| dict | None | Raw extraction spec to apply on the response (encoded and sent as |
| str | None | Name of a saved server-side extraction template |
| dict | None | Inline JSON extraction template used once ( |
| str | None | Natural language instructions for Extraction API |
| str | None | LLM model name to use for Extraction API |
| bool | False | Enable response caching |
| int | None | Cache time-to-live in seconds |
| bool | False | Clear any existing cached response when |
| bool | False | Collect DNS records and timings in |
| bool | False | Collect SSL certificate details in |
| bool | False | Enable debug recording and extra metadata |
| bool | True | Raise exceptions for upstream 4xx/5xx HTTP responses |
| str | None | Custom ID for request tracking across systems |
| list[str] | None | Custom tags for request organization and analytics |
| list[str] | None | Accept-Language values, e.g. |
| str | None | Override browser OS fingerprint, e.g. |
| str | None | Named webhook for async scrape completion callbacks |
| int | None | Max credits to spend on unblocker retries and extra features |
response = client.scrape(ScrapeConfig(url="https://httpbin.dev"))
response.content # Page content (HTML/JSON/text)
response.scrape_result # Full result dict
response.status_code # Scrapfly API HTTP status code
response.scrape_result["status_code"] # Upsteam HTTP status code
response.headers # Response headers
response.context # Metadata about features usedfrom scrapfly import ScrapflyClient, ScrapeConfig
import os
client = ScrapflyClient(key=os.environ["SCRAPFLY_API_KEY"])
result = client.scrape(ScrapeConfig(
url="https://httpbin.dev",
))
print(result.content)result = client.scrape(ScrapeConfig(
url="https://httpbin.dev",
unblocker=True, # enable the unblocker to bypass antibots
country="us", # match the proxy with the domain country
proxy_pool="public_residential_pool", # use the residential proxy pool to match real ISP IPs
))result = client.scrape(ScrapeConfig(
url="https://web-scraping.dev/products",
render_js=True, # enable JavaScript rendering using a cloud browser
rendering_wait=5000, # rendering wait for wait for
wait_for_selector="div.product", # wait a for a selector to be present on the page
))result = client.scrape(ScrapeConfig(
url="https://web-scraping.dev/products",
format="markdown", # other supported formats are: json, text, clean_html
))
print(result.content)result = client.scrape(ScrapeConfig(
url="https://httpbin.dev/anything",
method="POST",
headers={"Content-Type": "application/json"},
body='{"query": "search term"}',
))
print(result.content)result = client.scrape(ScrapeConfig(
url="https://web-scraping.dev/login",
render_js=True,
# browser control
js_scenario=[
{
"fill":{
"selector":"input[name='username']","clear":True,"value":"user123"
}
},{
"fill":{
"selector":"input[name='password']","clear":True,"value":"password"
}
},{
"click":{
"selector":"form > button[type='submit']"
}
},{
"wait_for_navigation":{
"timeout":5000
}
}
],
# request headers
headers={
"cookie":"cookiesAccepted=true"
}
))
# access the element under login
print(result.selector.css("div#secret-message::text").get())result = client.scrape(ScrapeConfig(
url="https://web-scraping.dev/products",
render_js=True
))
# access the built-in Parsel selector
selector = result.selector
for product in selector.css("div.product"):
print("product name:", product.xpath(".//a/text()").get()) # using xpath
print("product price:", product.css("div.price::text").get()) # using cssimport asyncio
from scrapfly import ScrapflyClient, ScrapeConfig
client = ScrapflyClient(key=os.environ["SCRAPFLY_API_KEY"])
urls = [
"https://web-scraping.dev/products",
"https://web-scraping.dev/products?page=2",
"https://web-scraping.dev/products?page=3",
]
configs = [ScrapeConfig(url=url) for url in urls]
async def concurrent_scraping():
results = []
async for result in client.concurrent_scrape(configs):
results.append(result)
for result in results:
print(result.content[:100])
print("===========")
asyncio.run(concurrent_scraping())# First request: login
result = client.scrape(ScrapeConfig(
url="https://web-scraping.dev/login",
render_js=True,
# browser control
js_scenario=[
{
"fill":{
"selector":"input[name='username']","clear":True,"value":"user123"
}
},{
"fill":{
"selector":"input[name='password']","clear":True,"value":"password"
}
},{
"click":{
"selector":"form > button[type='submit']"
}
},{
"wait_for_navigation":{
"timeout":5000
}
}
],
# request headers
headers={
"cookie":"cookiesAccepted=true"
},
session="logged-in-session"
))
# Second request: access protected page with same session
result = client.scrape(ScrapeConfig(
url="https://web-scraping.dev/login",
session="logged-in-session"
))
# access the built-in selector
print(result.selector.css("div#secret-message::text").get())result = client.scrape(ScrapeConfig(
url="https://example.com",
cache=True,
cache_ttl=3600, # Cache for 1 hour
))from scrapfly import ScrapflyClient, ScrapeConfig
from scrapfly.errors import (
ScrapflyError,
UpstreamHttpClientError,
UpstreamHttpServerError,
ScrapflyProxyError,
ScrapflyThrottleError,
)
try:
result = client.scrape(ScrapeConfig(url="https://httpbin.dev", unblocker=True))
except ScrapflyThrottleError as e:
print(f"Rate limited, retry after {e.retry_delay}s")
except UpstreamHttpClientError as e:
print(f"Target returned 4xx: {e.message}")
except UpstreamHttpServerError as e:
print(f"Target returned 5xx: {e.message}")
except ScrapflyProxyError as e:
print(f"Proxy error: {e.message}")
except ScrapflyError as e:
print(f"Scrapfly error: {e.message}")render_js=Truerendering_waitwait_for_selectorjsjs_scenarioscreenshotsproxy_pool="public_residential_pool"unblocker=Trueunblockeraspaspformat="markdown"sessionconcurrent_scrape