Loading...
Loading...
Crawl entire websites using the Scrapfly Crawler API with the Python SDK
npx skill4agent add scrapfly/skills scrapfly-crawlerpip install scrapfly-sdkSCRAPFLY_API_KEYfrom scrapfly import ScrapflyClient, CrawlerConfig, Crawlfrom scrapfly import (
webhook_from_payload,
CrawlStartedWebhook,
CrawlUrlDiscoveredWebhook,
CrawlUrlFailedWebhook,
CrawlCompletedWebhook,
)| Parameter | Type | Default | Description |
|---|---|---|---|
| str | required | Starting URL for the crawl |
| int | None | Maximum number of pages to crawl |
| int | None | Maximum link traversal depth from start URL |
| int | None | Maximum crawl duration in seconds |
| list[str] | None | URL path patterns to skip (supports |
| list[str] | None | Only crawl URLs matching these path patterns (supports |
| bool | False | Allow crawling any path on the same domain, not just the seed URL's base path |
| bool | False | Follow links to external domains |
| list[str] | None | Whitelist of external domains when |
| dict | None | Custom HTTP headers as JSON object |
| int | None | Delay between requests in milliseconds (0-15000) |
| str | None | Custom User-Agent string (ignored when |
| int | None | Maximum concurrent crawl requests |
| int | None | Wait time in ms after page load before extraction (0-25000). |
| bool | False | Discover URLs from |
| bool | False | Respect |
| bool | False | Ignore |
| bool | False | Enable cache layer for crawled pages |
| int | None | Cache time-to-live in seconds (0-604800) |
| bool | False | Force refresh of cached pages |
| list[str] | None | Content formats to extract: |
| dict | None | Custom extraction rules for structured data |
| bool | False | Enable the Unblocker anti-bot bypass with browser rendering. |
| str | None | Proxy pool to use (e.g., |
| str | None | Proxy country selection (ISO code) |
| str | None | Name of webhook configured in dashboard |
| list[str] | None | Events to subscribe to (e.g., |
| int | None | Maximum API credits to spend on this crawl |
"crawler_started""crawler_url_visited""crawler_url_skipped""crawler_url_discovered""crawler_url_failed""crawler_stopped""crawler_cancelled""crawler_finished"import os
from scrapfly import ScrapflyClient, CrawlerConfig, Crawl
client = ScrapflyClient(key=os.environ["SCRAPFLY_API_KEY"])
# Create, start, and wait for crawl in one chain
# 2. Create and run crawler
crawl = Crawl(
client,
CrawlerConfig(url='https://web-scraping.dev/products', page_limit=5)
).crawl().wait()
# 3. Get results
pages = crawl.warc().get_pages()
# 4. Process results
print(f"Crawled {len(pages)} pages:")
for page in pages:
print(f" • {page['url']} ({page['status_code']})")
# 5. Access specific URLs
html = crawl.read('https://web-scraping.dev/products')
if html:
print(f"\nMain page is {len(html):,} bytes")import os
import time
from scrapfly import ScrapflyClient, CrawlerConfig
client = ScrapflyClient(key=os.environ["SCRAPFLY_API_KEY"])
config = CrawlerConfig(
url='https://web-scraping.dev',
page_limit=10,
max_depth=2,
content_formats=['html', 'markdown']
)
# 1. Start crawler
start_response = client.start_crawl(config)
print(f"Crawler started: {start_response.uuid}")
# 2. Poll for status
while True:
status = client.get_crawl_status(start_response.uuid)
print(f"Status: {status.status} - {status.progress_pct:.1f}%")
print(f"Crawled: {status.urls_crawled}/{status.urls_discovered} pages")
if status.is_complete:
break
elif status.is_failed:
print("Crawl failed!")
break
time.sleep(5)
# 3. Download and process results
if status.is_complete:
artifact = client.get_crawl_artifact(start_response.uuid)
pages = artifact.get_pages()
print(f"Downloaded {len(pages)} pages")
for page in pages:
print(f" {page['url']}: {page['status_code']} ({len(page['content'])} bytes)")
# Save to file
artifact.save('crawl_results.warc.gz')config = CrawlerConfig(
url='https://web-scraping.dev/',
page_limit=50,
max_depth=5,
include_only_paths=['*/product/*'],
content_formats=['markdown', 'text'],
)
crawl = Crawl(client, config).crawl().wait()config = CrawlerConfig(
url='https://web-scraping.dev',
page_limit=50,
unblocker=True,
content_formats=['html'],
)
crawl = Crawl(client, config).crawl().wait()config = CrawlerConfig(
url='https://web-scraping.dev/products',
page_limit=100,
max_depth=3,
include_only_paths=['/products/*'],
extraction_rules={
"/products/*": {
"type": "model",
"value": "product",
},
},
)
crawl = Crawl(client, config).crawl().wait()# Read HTML (from WARC - fast)
content = crawl.read('https://web-scraping.dev/product/1')
if content:
print(f"URL: {content.url}")
print(f"Status: {content.status_code}")
print(f"Duration: {content.duration}s")
print(content.content)
# Read markdown (from contents API)
content = crawl.read('https://web-scraping.dev/product/1', format='markdown')
if content:
print(content.content)for content in crawl.read_iter(pattern="*/products?page=*", format="markdown"):
print(f"{content.url}: {len(content.content)} chars")# Get markdown for multiple URLs in a single API call (max 100 per request)
urls = ['https://web-scraping.dev/product/1', 'https://web-scraping.dev/product/2']
contents = crawl.read_batch(urls, formats=['markdown'])
for url, formats_dict in contents.items():
markdown = formats_dict.get('markdown', '')
print(f"{url}: {len(markdown)} chars")# WARC format (default)
artifact = crawl.warc()
artifact.save('crawl_results.warc.gz')
# HAR format
artifact = crawl.har()
artifact.save('crawl_results.har')
# Iterate through records
for record in artifact.iter_responses():
print(f"{record.url}: {record.status_code}")import os
from scrapfly import ScrapflyClient, CrawlerConfig, Crawl
client = ScrapflyClient(key=os.environ["SCRAPFLY_API_KEY"])
config = CrawlerConfig(
url='https://web-scraping.dev/',
include_only_paths=['/docs', '/docs/*'], # only allow doc resources
page_limit=50,
max_depth=5,
follow_external_links=False,
content_formats=['markdown'],
)
crawl = Crawl(client, config).crawl().wait(poll_interval=5, verbose=True)
# Get URLs from WARC
warc_artifact = crawl.warc()
all_urls = [
record.url for record in warc_artifact.iter_responses()
if record.status_code == 200
]
# Retrieve markdown content in batches
all_contents = {}
for i in range(0, len(all_urls), 100):
batch_urls = all_urls[i:i + 100]
batch_contents = crawl.read_batch(batch_urls, formats=['markdown'])
all_contents.update(batch_contents)
# Build llms-full.txt
lines = ["# Documentation\n"]
for url, formats_dict in all_contents.items():
markdown = formats_dict.get('markdown', '').strip()
if markdown:
lines.append(f"---\n\n### {url}\n\n{markdown}\n")
with open("llms-full.txt", "w", encoding="utf-8") as f:
f.write("\n".join(lines))stats = crawl.stats()
print(f"URLs discovered: {stats['urls_discovered']}")
print(f"URLs crawled: {stats['urls_crawled']}")
print(f"Progress: {stats['progress_pct']:.1f}%")| Class | Description |
|---|---|
| Main client - handles auth and API calls |
| Crawl job configuration |
| High-level crawl lifecycle manager (start, wait, read) |
| Response from |
| Response from |
| Response from |
| Content for a single URL - has |
page_limitinclude_only_pathsexclude_pathsexclude_pathsinclude_only_pathsCrawl.crawl().wait()client.get_crawl_status()crawl.warc()crawl.har()read_batch()follow_external_links