curl --request GET \
--url https://api.brightdata.com/datasets/v3/scrapers \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.brightdata.com/datasets/v3/scrapers"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.brightdata.com/datasets/v3/scrapers', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/datasets/v3/scrapers",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/datasets/v3/scrapers"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.brightdata.com/datasets/v3/scrapers")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/datasets/v3/scrapers")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body[
{
"id": "gd_mk57m0301khq4jmsul",
"name": "npmjs",
"description": null,
"domain": "npmjs.com",
"scraper_type": [
"collect_by_url"
],
"scrapers": {
"collect_by_url": {
"input_schema": [
{
"name": "package_name",
"type": "text",
"required": true
}
],
"output_fields": [
{
"name": "url",
"type": "url"
},
{
"name": "package_name",
"type": "text",
"description": "NPMjs package name"
},
{
"name": "version",
"type": "text",
"description": "NPMjs version number"
},
{
"name": "readme_content",
"type": "text",
"description": "NPMjs README.md content"
}
],
"sample_input": [
{
"package_name": "@brightdata/sdk"
}
],
"sample_output": {},
"link": "https://brightdata.com/cp/scrapers/gd_mk57m0301khq4jmsul/pdp/configuration"
}
}
}
]Get all scrapers
List every scraper on your Bright Data account. GET /datasets/v3/scrapers returns each scraper’s dataset ID, domain, input schema and output fields.
curl --request GET \
--url https://api.brightdata.com/datasets/v3/scrapers \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.brightdata.com/datasets/v3/scrapers"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.brightdata.com/datasets/v3/scrapers', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/datasets/v3/scrapers",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/datasets/v3/scrapers"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.brightdata.com/datasets/v3/scrapers")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/datasets/v3/scrapers")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body[
{
"id": "gd_mk57m0301khq4jmsul",
"name": "npmjs",
"description": null,
"domain": "npmjs.com",
"scraper_type": [
"collect_by_url"
],
"scrapers": {
"collect_by_url": {
"input_schema": [
{
"name": "package_name",
"type": "text",
"required": true
}
],
"output_fields": [
{
"name": "url",
"type": "url"
},
{
"name": "package_name",
"type": "text",
"description": "NPMjs package name"
},
{
"name": "version",
"type": "text",
"description": "NPMjs version number"
},
{
"name": "readme_content",
"type": "text",
"description": "NPMjs README.md content"
}
],
"sample_input": [
{
"package_name": "@brightdata/sdk"
}
],
"sample_output": {},
"link": "https://brightdata.com/cp/scrapers/gd_mk57m0301khq4jmsul/pdp/configuration"
}
}
}
]Authorizations
Use your Bright Data API Key as a Bearer token in the Authorization header.
How to authenticate:
- Obtain your API Key from the Bright Data account settings at https://brightdata.com/cp/setting/users
- Include the API Key in the Authorization header of your requests
- Format:
Authorization: Bearer YOUR_API_KEY
Example:
Authorization: Bearer YOUR_API_KEY
Learn how to get your Bright Data API key: https://docs.brightdata.com/api-reference/authentication
Query Parameters
Return only the scraper with this dataset ID. Omit it to return every scraper on the account.
"gd_l7q7dkf244hwjntr0"
Return only scrapers whose domain matches this value. Omit it to return every scraper on the account.
"amazon.com"
Response
OK
Dataset ID of the scraper. Pass it as the dataset_id query parameter on the trigger, scrape and snapshot endpoints.
"gd_l7q7dkf244hwjntr0"
Display name of the scraper, as shown in the Bright Data control panel.
"Amazon products"
Target site the scraper collects from. Filter on this field to group scrapers by site family.
"amazon.com"
Collection methods this scraper supports. collect_by_url collects a record directly from a URL or another identifier. Each discover_by_* method runs a discovery phase first, then collects what it finds. To run one, send type=discover_new and the method name without its discover_by_ prefix as discover_by on POST /datasets/v3/trigger, so discover_by_keyword becomes discover_by=keyword. The values match the keys of the scrapers object exactly.
["collect_by_url", "discover_by_keyword"]
Per-method detail keyed by collection method name. The keys match the scraper_type array exactly.
Show child attributes
Show child attributes
Overview of the data the scraper returns. Returns null, or is omitted, when the scraper has no published description.
Marketplace category. Returned values are ecomm, social, business, travel, real_estate, ad_verification, classifieds, financial, geospatial, AI Search and Data for AI. Omitted for uncategorized scrapers.
"ecomm"
Record count of the matching Bright Data marketplace dataset. Returned only for scrapers that have one.
300000000
Curated subsets of the matching marketplace dataset. Returns null, or is omitted, when the scraper publishes no subsets.
Show child attributes
Show child attributes
Was this page helpful?