Custom Headers

If you need to pass a specific parameter to the target web site, you can pass the headers to our endpoints just the same way.

To use this feature, you need to set customHeaders parameter to true.

Note that we take care of required headers such as user agents, cookies etc. you should not worry about these. But if you still want to pass those headers, the headers that you sent will be passed to client side since they are prioritized.

Bash
Python
Go
C#
Java
NodeJs
Javascript
Ruby
PHP
Swift
Bash
curl --header "test-header: value" \
"http://api.scrape.do?token=API_TOKEN
&url=http://example.com
&customHeaders=true"
Python
import requests
url = "http://api.scrape.do?token=API_TOKEN&url=http://example.com&customHeaders=true"
payload = {}
headers = {
'test-header': 'value'
}
response = requests.request("GET", url, headers=headers, data = payload)
print(response.text.encode('utf8'))
Go
package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "http://api.scrape.do?token=API_TOKEN&url=http://example.com&customHeaders=true"
method := "GET"
client := &http.Client {
}
req, err := http.NewRequest(method, url, nil)
if err != nil {
fmt.Println(err)
}
req.Header.Add("test-header", "value")
res, err := client.Do(req)
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
fmt.Println(string(body))
}
C#
var client = new RestClient("http://api.scrape.do?token=API_TOKEN&url=http://example.com&customHeaders=true");
client.Timeout = -1;
var request = new RestRequest(Method.GET);
request.AddHeader("test-header", "value");
IRestResponse response = client.Execute(request);
Console.WriteLine(response.Content);
Java
OkHttpClient client = new OkHttpClient().newBuilder()
.build();
Request request = new Request.Builder()
.url("http://api.scrape.do?token=API_TOKEN&url=http://example.com&customHeaders=true")
.method("GET", null)
.addHeader("test-header", "value")
.build();
Response response = client.newCall(request).execute();
NodeJs
var axios = require('axios');
var config = {
method: 'get',
url: 'http://api.scrape.do?token=API_TOKEN&url=http://example.com&customHeaders=true',
headers: {
'test-header': 'value'
}
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
Javascript
var xhr = new XMLHttpRequest();
xhr.withCredentials = true;
xhr.addEventListener("readystatechange", function() {
if(this.readyState === 4) {
console.log(this.responseText);
}
});
xhr.open("GET", "http://api.scrape.do?token=API_TOKEN&url=http://example.com&customHeaders=true");
xhr.setRequestHeader("test-header", "value");
xhr.send();
Ruby
require "uri"
require "net/http"
url = URI("http://api.scrape.do?token=API_TOKEN&url=http://example.com&customHeaders=true")
http = Net::HTTP.new(url.host, url.port);
request = Net::HTTP::Get.new(url)
request["test-header"] = "value"
response = http.request(request)
puts response.read_body
PHP
<?php
$curl = curl_init();
curl_setopt_array($curl, array(
CURLOPT_URL => "http://api.scrape.do?token=API_TOKEN&url=http://example.com&customHeaders=true",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 0,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => array(
"test-header: value"
),
));
$response = curl_exec($curl);
curl_close($curl);
echo $response;
Swift
import Foundation
var semaphore = DispatchSemaphore (value: 0)
var request = URLRequest(url: URL(string: "http://api.scrape.do?token=API_TOKEN&url=http://example.com&customHeaders=true")!,timeoutInterval: Double.infinity)
request.addValue("value", forHTTPHeaderField: "test-header")
request.httpMethod = "GET"
let task = URLSession.shared.dataTask(with: request) { data, response, error in
guard let data = data else {
print(String(describing: error))
return
}
print(String(data: data, encoding: .utf8)!)
semaphore.signal()
}
task.resume()
semaphore.wait()

Result

<!doctype html>
<html>
<head>
<title>Example Domain</title>
<meta charset="utf-8" />
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
..............
..............
</head>
</html>