Headless Browsers (JS Render)

Use actual browsers so that you can crawl any website you want without getting banned.
Javascript render with real browsers

Scrape.do offers headless browser service using actual Chrome browsers. Unlike other API services, it does not use tools such as Selenium, it reveals the power of actual browsers by scraping the target site without getting banned. With using this way, the target site cannot detect if it is an actual person surfing on the web or scrape.do crawlers.

  • You may need to use headless mode if you get banned.

  • Using React, Angular, Vue.js or other libraries, we create Javascript with a simple parameter so that you can crawl each website or even a single page application easily.

  • Javascript requests are billed as 10 credits for each successful API call.

To use Javascript, all you need to do is setting render parameter to true.

Beware that you need a business plan to use this feature!

Bash
Python
Go
C#
Java
NodeJs
Javascript
Ruby
PHP
Swift
Bash
$ curl "http://api.scrape.do?token=API_TOKEN&render=true \
&url=https://www.example.com"
Python
import requests
url = "http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com"
payload = {}
headers= {}
response = requests.request("GET", url, headers=headers, data = payload)
print(response.text.encode('utf8'))
Go
package main
import (
"fmt"
"net/http"
"io/ioutil"
)
func main() {
url := "http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com"
method := "GET"
client := &http.Client {
}
req, err := http.NewRequest(method, url, nil)
if err != nil {
fmt.Println(err)
}
res, err := client.Do(req)
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
fmt.Println(string(body))
}
C#
var client = new RestClient("http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com");
client.Timeout = -1;
var request = new RestRequest(Method.GET);
IRestResponse response = client.Execute(request);
Console.WriteLine(response.Content);
Java
OkHttpClient client = new OkHttpClient().newBuilder()
.build();
Request request = new Request.Builder()
.url("http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com")
.method("GET", null)
.build();
Response response = client.newCall(request).execute();
NodeJs
var axios = require('axios');
var config = {
method: 'get',
url: 'http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com',
headers: { }
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
Javascript
var xhr = new XMLHttpRequest();
xhr.withCredentials = true;
xhr.addEventListener("readystatechange", function() {
if(this.readyState === 4) {
console.log(this.responseText);
}
});
xhr.open("GET", "http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com");
xhr.send();
Ruby
require "uri"
require "net/http"
url = URI("http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com")
http = Net::HTTP.new(url.host, url.port);
request = Net::HTTP::Get.new(url)
response = http.request(request)
puts response.read_body
PHP
<?php
$curl = curl_init();
curl_setopt_array($curl, array(
CURLOPT_URL => "http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 0,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
));
$response = curl_exec($curl);
curl_close($curl);
echo $response;
Swift
import Foundation
var semaphore = DispatchSemaphore (value: 0)
var request = URLRequest(url: URL(string: "http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com")!,timeoutInterval: Double.infinity)
request.httpMethod = "GET"
let task = URLSession.shared.dataTask(with: request) { data, response, error in
guard let data = data else {
print(String(describing: error))
return
}
print(String(data: data, encoding: .utf8)!)
semaphore.signal()
}
task.resume()
semaphore.wait()

Result

<!doctype html>
<html>
<head>
<title>Example Domain</title>
<meta charset="utf-8" />
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
..............
..............
</head>
</html>