Deliver a snapshot to Amazon S3, Azure Blob Storage or a Webhook
curl --request POST \
--url https://api.brightdata.com/webarchive/dump \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"search_id": "<string>",
"delivery": {
"strategy": "s3",
"settings": {
"bucket": "<string>",
"assume_role": {
"role_arn": "<string>"
},
"prefix": "<string>"
}
},
"max_entries": 123
}
'import requests
url = "https://api.brightdata.com/webarchive/dump"
payload = {
"search_id": "<string>",
"delivery": {
"strategy": "s3",
"settings": {
"bucket": "<string>",
"assume_role": { "role_arn": "<string>" },
"prefix": "<string>"
}
},
"max_entries": 123
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
search_id: '<string>',
delivery: {
strategy: 's3',
settings: {bucket: '<string>', assume_role: {role_arn: '<string>'}, prefix: '<string>'}
},
max_entries: 123
})
};
fetch('https://api.brightdata.com/webarchive/dump', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/webarchive/dump",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'search_id' => '<string>',
'delivery' => [
'strategy' => 's3',
'settings' => [
'bucket' => '<string>',
'assume_role' => [
'role_arn' => '<string>'
],
'prefix' => '<string>'
]
],
'max_entries' => 123
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/webarchive/dump"
payload := strings.NewReader("{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/webarchive/dump")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/webarchive/dump")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}"
response = http.request(request)
puts response.read_body{
"dump_id": "ucd_abc123-1234567890"
}{
"error": "Request validation failed",
"error_code": "validation",
"details": [
{
"message": "\"search_id\" is required",
"path": [
"search_id"
],
"type": "any.required"
}
]
}Web Archive API
Deliver snapshot to S3, Azure Blob or GCP
Deliver a Bright Data Archive API snapshot to Amazon S3, Azure Blob Storage, Google Cloud Storage or a webhook. POST /webarchive/dump returns a dump_id.
POST
/
webarchive
/
dump
Deliver a snapshot to Amazon S3, Azure Blob Storage or a Webhook
curl --request POST \
--url https://api.brightdata.com/webarchive/dump \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"search_id": "<string>",
"delivery": {
"strategy": "s3",
"settings": {
"bucket": "<string>",
"assume_role": {
"role_arn": "<string>"
},
"prefix": "<string>"
}
},
"max_entries": 123
}
'import requests
url = "https://api.brightdata.com/webarchive/dump"
payload = {
"search_id": "<string>",
"delivery": {
"strategy": "s3",
"settings": {
"bucket": "<string>",
"assume_role": { "role_arn": "<string>" },
"prefix": "<string>"
}
},
"max_entries": 123
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
search_id: '<string>',
delivery: {
strategy: 's3',
settings: {bucket: '<string>', assume_role: {role_arn: '<string>'}, prefix: '<string>'}
},
max_entries: 123
})
};
fetch('https://api.brightdata.com/webarchive/dump', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/webarchive/dump",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'search_id' => '<string>',
'delivery' => [
'strategy' => 's3',
'settings' => [
'bucket' => '<string>',
'assume_role' => [
'role_arn' => '<string>'
],
'prefix' => '<string>'
]
],
'max_entries' => 123
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/webarchive/dump"
payload := strings.NewReader("{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/webarchive/dump")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/webarchive/dump")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"search_id\": \"<string>\",\n \"delivery\": {\n \"strategy\": \"s3\",\n \"settings\": {\n \"bucket\": \"<string>\",\n \"assume_role\": {\n \"role_arn\": \"<string>\"\n },\n \"prefix\": \"<string>\"\n }\n },\n \"max_entries\": 123\n}"
response = http.request(request)
puts response.read_body{
"dump_id": "ucd_abc123-1234567890"
}{
"error": "Request validation failed",
"error_code": "validation",
"details": [
{
"message": "\"search_id\" is required",
"path": [
"search_id"
],
"type": "any.required"
}
]
}POST /webarchive/dump delivers the snapshot from a completed search to Amazon S3, Azure Blob Storage, Google Cloud Storage or a webhook, and returns a dump_id.
To use S3 storage delivery, you will first need to do the following:
- Create an AWS role which gives Bright Data access to your system.
- During this setup, you will be asked by Amazon for an “external ID” that is used with the role.
- Your external ID for S3 is your Bright Data Account ID that can be found within Account Settings
- Once a role is created, you will need to allow the Bright Data delivery role to
AssumeRolethat role.- The Bright Data delivery role is:
arn:aws:iam::422310177405:role/brd.ec2.zs-dca-delivery
- The Bright Data delivery role is:
To use Google Cloud Storage delivery, create a bucket and provide the required GCP delivery settings.
The webhook delivery strategy is not suitable for large dumps unless you
are hosting the webhook on your own infrastructure. Third-party inspection
tools such as webhook.site impose strict request body
size limits and will fail to receive payloads that can reach up to 1 GB in
size. For large deliveries, use Amazon S3, Azure Blob Storage
or Google Cloud Storage instead.
Common dump parameters:
search_id(required): The search ID from a completed searchmax_entries(optional): Limit the number of files to include in the dumpdelivery(required): Delivery configuration (S3, Azure, GCP, or webhook)
If you’re running a linux/macos machine, you can simulate one of our delivery webhooks with the code on this page.
What causes a 400 response
POST /webarchive/dump returns HTTP 400 when the request body fails validation. The response carries an error summary and a details array naming each field that failed. Common causes:
search_idis missing.deliveryis missing.- The
delivery.settingsblock does not match the chosendelivery.strategy. Each strategy has its own required settings:bucketplusassume_rolefor Amazon S3,bucketfor Google Cloud Storage,containerpluscredentialsfor Azure Blob Storage andurlfor a webhook.
details[].path and resend the request.Authorizations
Use your Bright Data API Key as a Bearer token in the Authorization header.
How to authenticate:
- Obtain your API Key from the Bright Data account settings at https://brightdata.com/cp/setting/users
- Include the API Key in the Authorization header of your requests
- Format:
Authorization: Bearer YOUR_API_KEY
Example:
Authorization: Bearer b5648e1096c6442f60a6c4bbbe73f8d2234d3d8324554bd6a7ec8f3f251f07df
Learn how to get your Bright Data API key: https://docs.brightdata.com/api-reference/authentication
Body
application/json
Response
Dump created successfully
ID of the created dump
Example:
"ucd_abc123-1234567890"
Was this page helpful?