Loading...
Loading...
Extract structured data from web content using the Scrapfly Extraction API with the Python SDK
npx skill4agent add scrapfly/skills scrapfly-extractionpip install scrapfly-sdkSCRAPFLY_API_KEYPOST https://api.scrapfly.io/extractionfrom scrapfly import ScrapflyClient, ExtractionConfig
import os
client = ScrapflyClient(key=os.environ["SCRAPFLY_API_KEY"])| Parameter | Type | Default | Description |
|---|---|---|---|
| str | required | Document content to extract from |
| str | required | Input format: |
| str | None | Base URL for resolving relative links in HTML |
| str | None | Document encoding (auto-detected if omitted) |
| str | None | Natural language instruction for LLM extraction |
| str | None | Pre-trained model: |
| str | None | Custom template name or inline template definition |
| int | None | Processing timeout in seconds (60-155) |
| str | None | Webhook name for async processing |
extraction_promptextraction_modelextraction_template"article""event""food_recipe""hotel""hotel_listing""job_listing""job_posting""organization""product""product_listing""real_estate_property""real_estate_property_listing""review_list""search_engine_results""social_media_post""software""stock""vehicle_ad""vehicle_ad_listing"result = client.extract(ExtractionConfig(
body=html_content,
content_type="text/html",
url="https://example.com/product/1",
extraction_template="product-card",
))match_domainmatch_pathERR::EXTRACTION::TEMPLATE_URL_MISMATCHextraction_template = {
"source": "html",
"selectors": [
{
"name": "title",
"query": "h3.product-title::text",
"type": "css",
"formatters": [
{
"name": "uppercase"
}
],
},
{
"name": "description",
"query": "p.product-description::text",
"type": "css"
},
{
"extractor": {
"name": "price"
},
"name": "price",
"query": ".product-price::text",
"type": "css"
},
{
"name": "variants",
"query": "div.variants",
"type": "css",
"nested": [
{
"name": "name",
"query": "//a[@data-variant-id]/@data-variant-id",
"type": "xpath",
"multiple": True,
},
{
"name": "link",
"query": "//a[@data-variant-id]/@href",
"type": "xpath",
"multiple": True,
},
]
},
{
"name": "reviews",
"query": "div.review>p::text",
"type": "css",
"multiple": True,
}
]
}from scrapfly import ScrapflyClient, ExtractionConfig
import os
client = ScrapflyClient(key=os.environ["SCRAPFLY_API_KEY"])
html_content = "<html><body><h1>iPhone 15</h1><span class='price'>$999</span><p>Latest Apple smartphone</p></body></html>"
result = client.extract(ExtractionConfig(
body=html_content,
content_type="text/html",
extraction_prompt="Extract the product name, price, and description as JSON",
))
# result
result.extraction_result['data']
# or
print(result.data)
# result content_type
result.extraction_result['content_type']
# or
print(result.content_type)markdown_content = """
# Best Restaurants in NYC
1. **Le Bernardin** - French, $$$, 4.8 stars
2. **Peter Luger** - Steakhouse, $$$, 4.5 stars
3. **Di Fara Pizza** - Italian, $, 4.7 stars
"""
result = client.extract(ExtractionConfig(
body=markdown_content,
content_type="text/markdown",
extraction_prompt="Extract each restaurant as a JSON array with name, cuisine, price_range, and rating fields",
))
print(result.data)from scrapfly import ScrapflyClient, ExtractionConfig, ScrapeConfig
# First scrape the page, then extract
scrape_result = client.scrape(ScrapeConfig(url="https://web-scraping.dev/product/1"))
result = client.extract(ExtractionConfig(
body=scrape_result.content,
content_type="text/html",
url="https://web-scraping.dev/product/1",
extraction_model="product",
))
print(result.data)
# Returns: {"name": "...", "price": "...", "currency": "...", "description": "...", ...}result = client.extract(ExtractionConfig(
body=html_content,
content_type="text/html",
extraction_prompt="What is the most expensive item on this page and how much does it cost?",
))
print(result.data)from scrapfly import ScrapflyClient, ScrapeConfig, ExtractionConfig
import os
client = ScrapflyClient(key=os.environ["SCRAPFLY_API_KEY"])
# Step 1: Scrape the page
scrape_result = client.scrape(ScrapeConfig(
url="https://web-scraping.dev/product/1",
format="markdown",
))
# Step 2: Extract structured data from the content
extraction_result = client.extract(ExtractionConfig(
body=scrape_result.content,
content_type="text/markdown",
extraction_prompt="Extract all products as a JSON array with fields: name, price, availability",
))
print(extraction_result.data)result = client.scrape(ScrapeConfig(
url="https://web-scraping.dev/product/1",
extraction_prompt="Extract the product name, price, and description as JSON",
))
# Extraction result is included in the scrape response
print(result.scrape_result["extracted_data"])# First, scrape the web page to retrieve its HTML
api_response = client.scrape(scrape_config=ScrapeConfig(
url='https://web-scraping.dev/product/1',
render_js=True
))
html = api_response.content
# extraction template for HTML parsing instructions. It accepts the following:
# selectors: CSS, XPath, JMESPath, Regex, Nested (nesting multiple selector types)
# extractors: extracts commonly accessed data types: price, image, links, emails
# formatters: transforms the extracted data for common methods: lowercase, uppercase, datatime, etc.
# refer to the docs for more details: https://scrapfly.io/docs/extraction-api/rules-and-template#rules
extraction_template = {
"source": "html",
"selectors": [
{
"name": "title",
"query": "h3.product-title::text",
"type": "css",
"formatters": [
{
"name": "uppercase"
}
],
},
{
"name": "description",
"query": "p.product-description::text",
"type": "css"
},
{
"extractor": {
"name": "price"
},
"name": "price",
"query": ".product-price::text",
"type": "css"
},
{
"name": "variants",
"query": "div.variants",
"type": "css",
"nested": [
{
"name": "name",
"query": "//a[@data-variant-id]/@data-variant-id",
"type": "xpath",
"multiple": True,
},
{
"name": "link",
"query": "//a[@data-variant-id]/@href",
"type": "xpath",
"multiple": True,
},
]
},
{
"name": "reviews",
"query": "div.review>p::text",
"type": "css",
"multiple": True,
}
]
}
extraction_api_response = client.extract(
extraction_config=ExtractionConfig(
body=html, # pass the HTML content
content_type='text/html', # content data type
charset='utf-8', # passed content charset, use `auto` if you aren't sure
extraction_ephemeral_template=extraction_template # declared template defintion or template name saved on the dashboard
)
)
# extracted data
print(extraction_api_response.data)
# extracted data content_type
print(extraction_api_response.content_type)from scrapfly.errors import ScrapflyError
try:
result = client.extract(ExtractionConfig(
body="<html><body><h1>iPhone 15</h1><span class='price'>$999</span><p>Latest Apple smartphone</p></body></html",
content_type="text/html",
extraction_prompt="Extract the product price",
))
print(result.data)
except ScrapflyError as e:
print(f"Extraction failed: {e.message}")extraction_promptextraction_modelextraction_templateurlformat="markdown"format="text"extraction_promptextraction_modelScrapeConfig