Scrape data and return it directly in the response.
curl --request POST \
--url https://api.brightdata.com/datasets/v3/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"custom_input_fields": [
"url",
"prompt",
"index_custom"
],
"input": [
{
"url": "https://chatgpt.com/",
"prompt": "Top hotels in New York",
"index_custom": "abd45424"
}
]
}
'import requests
url = "https://api.brightdata.com/datasets/v3/scrape"
payload = {
"custom_input_fields": ["url", "prompt", "index_custom"],
"input": [
{
"url": "https://chatgpt.com/",
"prompt": "Top hotels in New York",
"index_custom": "abd45424"
}
]
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
custom_input_fields: ['url', 'prompt', 'index_custom'],
input: [
{
url: 'https://chatgpt.com/',
prompt: 'Top hotels in New York',
index_custom: 'abd45424'
}
]
})
};
fetch('https://api.brightdata.com/datasets/v3/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/datasets/v3/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'custom_input_fields' => [
'url',
'prompt',
'index_custom'
],
'input' => [
[
'url' => 'https://chatgpt.com/',
'prompt' => 'Top hotels in New York',
'index_custom' => 'abd45424'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/datasets/v3/scrape"
payload := strings.NewReader("{\n \"custom_input_fields\": [\n \"url\",\n \"prompt\",\n \"index_custom\"\n ],\n \"input\": [\n {\n \"url\": \"https://chatgpt.com/\",\n \"prompt\": \"Top hotels in New York\",\n \"index_custom\": \"abd45424\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/datasets/v3/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"custom_input_fields\": [\n \"url\",\n \"prompt\",\n \"index_custom\"\n ],\n \"input\": [\n {\n \"url\": \"https://chatgpt.com/\",\n \"prompt\": \"Top hotels in New York\",\n \"index_custom\": \"abd45424\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/datasets/v3/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"custom_input_fields\": [\n \"url\",\n \"prompt\",\n \"index_custom\"\n ],\n \"input\": [\n {\n \"url\": \"https://chatgpt.com/\",\n \"prompt\": \"Top hotels in New York\",\n \"index_custom\": \"abd45424\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body"OK"{
"status": "starting",
"message": "Snapshot is not ready yet, try again in 30s"
}Scraper API
Custom inputs
Add custom fields to a Bright Data Web Scraper API input schema; values you send are returned in each output record for tagging.
POST
/
datasets
/
v3
/
scrape
Scrape data and return it directly in the response.
curl --request POST \
--url https://api.brightdata.com/datasets/v3/scrape \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"custom_input_fields": [
"url",
"prompt",
"index_custom"
],
"input": [
{
"url": "https://chatgpt.com/",
"prompt": "Top hotels in New York",
"index_custom": "abd45424"
}
]
}
'import requests
url = "https://api.brightdata.com/datasets/v3/scrape"
payload = {
"custom_input_fields": ["url", "prompt", "index_custom"],
"input": [
{
"url": "https://chatgpt.com/",
"prompt": "Top hotels in New York",
"index_custom": "abd45424"
}
]
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
custom_input_fields: ['url', 'prompt', 'index_custom'],
input: [
{
url: 'https://chatgpt.com/',
prompt: 'Top hotels in New York',
index_custom: 'abd45424'
}
]
})
};
fetch('https://api.brightdata.com/datasets/v3/scrape', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/datasets/v3/scrape",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'custom_input_fields' => [
'url',
'prompt',
'index_custom'
],
'input' => [
[
'url' => 'https://chatgpt.com/',
'prompt' => 'Top hotels in New York',
'index_custom' => 'abd45424'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/datasets/v3/scrape"
payload := strings.NewReader("{\n \"custom_input_fields\": [\n \"url\",\n \"prompt\",\n \"index_custom\"\n ],\n \"input\": [\n {\n \"url\": \"https://chatgpt.com/\",\n \"prompt\": \"Top hotels in New York\",\n \"index_custom\": \"abd45424\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/datasets/v3/scrape")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"custom_input_fields\": [\n \"url\",\n \"prompt\",\n \"index_custom\"\n ],\n \"input\": [\n {\n \"url\": \"https://chatgpt.com/\",\n \"prompt\": \"Top hotels in New York\",\n \"index_custom\": \"abd45424\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/datasets/v3/scrape")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"custom_input_fields\": [\n \"url\",\n \"prompt\",\n \"index_custom\"\n ],\n \"input\": [\n {\n \"url\": \"https://chatgpt.com/\",\n \"prompt\": \"Top hotels in New York\",\n \"index_custom\": \"abd45424\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body"OK"{
"status": "starting",
"message": "Snapshot is not ready yet, try again in 30s"
}Custom inputs
You can add custom fields to your input schema, and whatever you send in those fields will be returned in the results for each record/job. This is useful for:- Unified schema: Keep the same output structure across different scrapers/datasets.
- Index / reference fields: Pass an
id,row_index, or any internal key so you can easily match results back to the original input rows.
Authorizations
Use your Bright Data API Key as a Bearer token in the Authorization header.
How to authenticate:
- Obtain your API Key from the Bright Data account settings at https://brightdata.com/cp/setting/users
- Include the API Key in the Authorization header of your requests
- Format:
Authorization: Bearer YOUR_API_KEY
Example:
Authorization: Bearer b5648e1096c6442f60a6c4bbbe73f8d2234d3d8324554bd6a7ec8f3f251f07df
Learn how to get your Bright Data API key: https://docs.brightdata.com/api-reference/authentication
Query Parameters
Dataset ID for which data collection is triggered.
List of output columns, separated by | (e.g., url|about.updated_on). Filters the response to include only the specified fields.
Example:
"url|about.updated_on"
Include errors report with the results.
Specifies the format of the response (default: ndjson).
Available options:
ndjson, json, csv Body
application/json
List of input items to scrape.
Show child attributes
Show child attributes
Example:
[
{
"url": "https://chatgpt.com/",
"prompt": "Top hotels in New York",
"index_custom": "abd45424"
}
]
List of custom input field names whose values are passed through and returned unchanged in the results for each record.
The name of a custom input field to be accepted and returned in the results.
Example:
["url", "prompt", "index_custom"]
List of output columns, separated by | (e.g., url|about.updated_on). Filters the response to include only the specified fields.
Example:
"url|about.updated_on"
Response
OK
The response is of type string.
Example:
"OK"
Was this page helpful?