Scrape.do offers headless browser service using actual Chrome browsers. Unlike other API services, it does not use tools such as Selenium, it reveals the power of actual browsers by scraping the target site without getting banned. With using this way, the target site cannot detect if it is an actual person surfing on the web or scrape.do crawlers.
You may need to use headless mode if you get banned.
Using React, Angular, Vue.js or other libraries, we create Javascript with a simple parameter so that you can crawl each website or even a single page application easily.
Javascript requests are billed as 10 credits for each successful API call.
To use Javascript, all you need to do is setting render
parameter to true
.
Beware that you need a business plan to use this feature!
$ curl "http://api.scrape.do?token=API_TOKEN&render=true \&url=https://www.example.com"
import requestsurl = "http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com"payload = {}headers= {}response = requests.request("GET", url, headers=headers, data = payload)print(response.text.encode('utf8'))
package mainimport ("fmt""net/http""io/ioutil")func main() {url := "http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com"method := "GET"client := &http.Client {}req, err := http.NewRequest(method, url, nil)if err != nil {fmt.Println(err)}res, err := client.Do(req)defer res.Body.Close()body, err := ioutil.ReadAll(res.Body)fmt.Println(string(body))}
var client = new RestClient("http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com");client.Timeout = -1;var request = new RestRequest(Method.GET);IRestResponse response = client.Execute(request);Console.WriteLine(response.Content);
OkHttpClient client = new OkHttpClient().newBuilder().build();Request request = new Request.Builder().url("http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com").method("GET", null).build();Response response = client.newCall(request).execute();
var axios = require('axios');var config = {method: 'get',url: 'http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com',headers: { }};axios(config).then(function (response) {console.log(JSON.stringify(response.data));}).catch(function (error) {console.log(error);});
var xhr = new XMLHttpRequest();xhr.withCredentials = true;xhr.addEventListener("readystatechange", function() {if(this.readyState === 4) {console.log(this.responseText);}});xhr.open("GET", "http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com");xhr.send();
require "uri"require "net/http"url = URI("http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com")http = Net::HTTP.new(url.host, url.port);request = Net::HTTP::Get.new(url)response = http.request(request)puts response.read_body
<?php$curl = curl_init();curl_setopt_array($curl, array(CURLOPT_URL => "http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com",CURLOPT_RETURNTRANSFER => true,CURLOPT_ENCODING => "",CURLOPT_MAXREDIRS => 10,CURLOPT_TIMEOUT => 0,CURLOPT_FOLLOWLOCATION => true,CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,CURLOPT_CUSTOMREQUEST => "GET",));$response = curl_exec($curl);curl_close($curl);echo $response;
import Foundationvar semaphore = DispatchSemaphore (value: 0)var request = URLRequest(url: URL(string: "http://api.scrape.do?token=API_TOKEN&render=true&url=http://example.com")!,timeoutInterval: Double.infinity)request.httpMethod = "GET"let task = URLSession.shared.dataTask(with: request) { data, response, error inguard let data = data else {print(String(describing: error))return}print(String(data: data, encoding: .utf8)!)semaphore.signal()}task.resume()semaphore.wait()
<!doctype html><html><head><title>Example Domain</title><meta charset="utf-8" /><meta http-equiv="Content-type" content="text/html; charset=utf-8" />............................</head></html>