cURL
curl --request POST \
--url https://api.brightdata.com/datasets/filter \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: multipart/form-data' \
--form 'filter={
"name": "name",
"operator": "=",
"value": "John"
}' \
--form 'files=<string>' \
--form files.items='@example-file'import requests
url = "https://api.brightdata.com/datasets/filter"
files = { "files.items": ("example-file", open("example-file", "rb")) }
payload = {
"filter": "{
\"name\": \"name\",
\"operator\": \"=\",
\"value\": \"John\"
}",
"files": "<string>"
}
headers = {"Authorization": "Bearer <token>"}
response = requests.post(url, data=payload, files=files, headers=headers)
print(response.text)const form = new FormData();
form.append('filter', '{
"name": "name",
"operator": "=",
"value": "John"
}');
form.append('files', '<string>');
form.append('files.items', '{
"fileName": "example-file"
}');
const options = {method: 'POST', headers: {Authorization: 'Bearer <token>'}};
options.body = form;
fetch('https://api.brightdata.com/datasets/filter', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/datasets/filter",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => "-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"filter\"\r\n\r\n{\r\n \"name\": \"name\",\r\n \"operator\": \"=\",\r\n \"value\": \"John\"\r\n}\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files\"\r\n\r\n<string>\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files.items\"; filename=\"example-file\"\r\nContent-Type: application/octet-stream\r\n\r\n{\r\n \"fileName\": \"example-file\"\r\n}\r\n-----011000010111000001101001--",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: multipart/form-data"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/datasets/filter"
payload := strings.NewReader("-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"filter\"\r\n\r\n{\r\n \"name\": \"name\",\r\n \"operator\": \"=\",\r\n \"value\": \"John\"\r\n}\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files\"\r\n\r\n<string>\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files.items\"; filename=\"example-file\"\r\nContent-Type: application/octet-stream\r\n\r\n{\r\n \"fileName\": \"example-file\"\r\n}\r\n-----011000010111000001101001--")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/datasets/filter")
.header("Authorization", "Bearer <token>")
.body("-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"filter\"\r\n\r\n{\r\n \"name\": \"name\",\r\n \"operator\": \"=\",\r\n \"value\": \"John\"\r\n}\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files\"\r\n\r\n<string>\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files.items\"; filename=\"example-file\"\r\nContent-Type: application/octet-stream\r\n\r\n{\r\n \"fileName\": \"example-file\"\r\n}\r\n-----011000010111000001101001--")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/datasets/filter")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request.body = "-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"filter\"\r\n\r\n{\r\n \"name\": \"name\",\r\n \"operator\": \"=\",\r\n \"value\": \"John\"\r\n}\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files\"\r\n\r\n<string>\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files.items\"; filename=\"example-file\"\r\nContent-Type: application/octet-stream\r\n\r\n{\r\n \"fileName\": \"example-file\"\r\n}\r\n-----011000010111000001101001--"
response = http.request(request)
puts response.read_body{
"snapshot_id": "<string>"
}{
"validation_errors": [
"\"filter.filters[0].invalid_prop\" is not allowed",
"\"records_limit\" must be a positive number"
]
}{
"error": "Your current balance is insufficient to process this data collection request. Please add funds to your account or adjust your request to continue. ($1 is missing)"
}{
"error": "Provided filter did not match any records"
}{
"error": "Maximum limit of 100 jobs per dataset has been exceeded"
}Dataset API
Filter dataset (JSON or file uploads)
Use the Bright Data Marketplace Dataset API to filter Dataset (JSON or File Uploads). Spans 250+ domains in the Bright Data marketplace.
POST
/
datasets
/
filter
cURL
curl --request POST \
--url https://api.brightdata.com/datasets/filter \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: multipart/form-data' \
--form 'filter={
"name": "name",
"operator": "=",
"value": "John"
}' \
--form 'files=<string>' \
--form files.items='@example-file'import requests
url = "https://api.brightdata.com/datasets/filter"
files = { "files.items": ("example-file", open("example-file", "rb")) }
payload = {
"filter": "{
\"name\": \"name\",
\"operator\": \"=\",
\"value\": \"John\"
}",
"files": "<string>"
}
headers = {"Authorization": "Bearer <token>"}
response = requests.post(url, data=payload, files=files, headers=headers)
print(response.text)const form = new FormData();
form.append('filter', '{
"name": "name",
"operator": "=",
"value": "John"
}');
form.append('files', '<string>');
form.append('files.items', '{
"fileName": "example-file"
}');
const options = {method: 'POST', headers: {Authorization: 'Bearer <token>'}};
options.body = form;
fetch('https://api.brightdata.com/datasets/filter', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.brightdata.com/datasets/filter",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => "-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"filter\"\r\n\r\n{\r\n \"name\": \"name\",\r\n \"operator\": \"=\",\r\n \"value\": \"John\"\r\n}\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files\"\r\n\r\n<string>\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files.items\"; filename=\"example-file\"\r\nContent-Type: application/octet-stream\r\n\r\n{\r\n \"fileName\": \"example-file\"\r\n}\r\n-----011000010111000001101001--",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: multipart/form-data"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.brightdata.com/datasets/filter"
payload := strings.NewReader("-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"filter\"\r\n\r\n{\r\n \"name\": \"name\",\r\n \"operator\": \"=\",\r\n \"value\": \"John\"\r\n}\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files\"\r\n\r\n<string>\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files.items\"; filename=\"example-file\"\r\nContent-Type: application/octet-stream\r\n\r\n{\r\n \"fileName\": \"example-file\"\r\n}\r\n-----011000010111000001101001--")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.brightdata.com/datasets/filter")
.header("Authorization", "Bearer <token>")
.body("-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"filter\"\r\n\r\n{\r\n \"name\": \"name\",\r\n \"operator\": \"=\",\r\n \"value\": \"John\"\r\n}\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files\"\r\n\r\n<string>\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files.items\"; filename=\"example-file\"\r\nContent-Type: application/octet-stream\r\n\r\n{\r\n \"fileName\": \"example-file\"\r\n}\r\n-----011000010111000001101001--")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.brightdata.com/datasets/filter")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request.body = "-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"filter\"\r\n\r\n{\r\n \"name\": \"name\",\r\n \"operator\": \"=\",\r\n \"value\": \"John\"\r\n}\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files\"\r\n\r\n<string>\r\n-----011000010111000001101001\r\nContent-Disposition: form-data; name=\"files.items\"; filename=\"example-file\"\r\nContent-Type: application/octet-stream\r\n\r\n{\r\n \"fileName\": \"example-file\"\r\n}\r\n-----011000010111000001101001--"
response = http.request(request)
puts response.read_body{
"snapshot_id": "<string>"
}{
"validation_errors": [
"\"filter.filters[0].invalid_prop\" is not allowed",
"\"records_limit\" must be a positive number"
]
}{
"error": "Your current balance is insufficient to process this data collection request. Please add funds to your account or adjust your request to continue. ($1 is missing)"
}{
"error": "Provided filter did not match any records"
}{
"error": "Maximum limit of 100 jobs per dataset has been exceeded"
}The Filter endpoint of the Bright Data Marketplace Dataset API can filter a dataset against thousands of values stored in a CSV or JSON file. Upload one or more files in multipart mode and reference each filename in your filter.
File references work only with the operators
Paste your API key into the authorization field. To get an API key, Create an account and learn how to generate an API key.
How does file-based filtering work?
Use file uploads when you need to filter against large value lists, such as including or excluding 100k+ company IDs:- Upload CSV or JSON files in
multipart/form-datamode and reference each filename in your filter. - Each file holds up to 10,000 data rows, and the whole request can be up to 200 MiB.
- The Filter job runs asynchronously and returns a
snapshot_idto download once it completes.
How do I format the CSV or JSON file?
- CSV
- JSON
- First line must be a header matching the field name in your filter.
- Each following line contains a single value.
Example: industries.csv
industries:value
Accounting
Ad Network
Advertising
- Must be an array of objects, each with a key matching the field name in your filter.
Example industries.json
[
{"industries:value": "Accounting"},
{"industries:value": "Ad Network"},
{"industries:value": "Advertising"}
]
How do I reference a file in the filter?
When using file uploads, set the filter’svalue field to the filename:
Example
{
"operator": "and",
"filters": [
{
"name": "industries:value",
"operator": "includes",
"value": "industries.csv"
}
]
}
in, not_in, includes, not_includes, array_includes and not_array_includes. For the full operator table and field types, see the filter syntax reference.
Filter with multiple files
curl \
-H "Authorization: Bearer YOUR_API_KEY" \
-H "Content-Type: multipart/form-data" \
-F "files[]=@/path/to/industries.csv" \
-F "files[]=@/path/to/regions.csv" \
-F "filter={\"operator\":\"and\",\"filters\":[{\"name\":\"industries:value\",\"operator\":\"includes\",\"value\":\"industries.csv\"},{\"name\":\"region\",\"operator\":\"in\",\"value\":\"regions.csv\"}]}" \
"https://api.brightdata.com/datasets/filter?dataset_id=gd_l1vijqt9jfj7olije"
Troubleshoot file uploads
| Issue | Possible Solution |
|---|---|
| ”File not found” | Make sure the filename in your filter exactly matches the uploaded file name. |
| ”Invalid file format” | Check CSV header matches the filter field name, or ensure JSON is an array of objects. |
| ”Field not found” | Verify field exists in dataset. Use Get Dataset Metadata. |
Related
- Dataset API overview
- Filter dataset (async)
- Filter syntax reference
- Get Dataset List
- Get Dataset Metadata
Authorizations
Use your Bright Data API Key as a Bearer token in the Authorization header.
How to authenticate:
- Obtain your API Key from the Bright Data account settings at https://brightdata.com/cp/setting/users
- Include the API Key in the Authorization header of your requests
- Format:
Authorization: Bearer YOUR_API_KEY
Example:
Authorization: Bearer b5648e1096c6442f60a6c4bbbe73f8d2234d3d8324554bd6a7ec8f3f251f07df
Learn how to get your Bright Data API key: https://docs.brightdata.com/api-reference/authentication
Query Parameters
ID of the dataset to filter (required in multipart/form-data mode)
Example:
"gd_l1viktl72bvl7bjuj0"
Limit the number of records to be included in the snapshot
Example:
1000
Body
multipart/form-data
- Single field filter
- Filters group
- Single field filter w/out value
Show child attributes
Show child attributes
Example:
{
"name": "name",
"operator": "=",
"value": "John"
}
CSV or JSON files referenced by name in the filter, sent as multipart file parts. The cURL samples use the array-notation alias files[].
Response
Job of creating the snapshot successfully started
ID of the snapshot
Was this page helpful?