# Parameters

The API has the following parameters, only the token and url are mandatory, the rest are optional.

# token

• Required
• Type string

This parameter is required for all calls

This is your authentication token. You have two tokens; one for normal requests and another one for JavaScript requests.

Use the JavaScript token when the content you need to crawl is generated via JavaScript, either because it's a JavaScript built page (React, Angular, etc.) or because the content is dynamically generated on the browser.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))

# url

• Required
• Type string

This parameter is required for all calls

You will need a url to crawl. Make sure it starts with http or https and that is fully encoded.

For example, in the following url: https://github.com/crawlbase?tab=repositories the url should be encoded when calling the API like the following: https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))

# format

• Optional
• Type string

Indicates the response format, either json or html. Defaults to html.

If format html is used, Crawlbase will send you back the response parameters in the headers (see HTML response below).

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories&format=json"

require 'net/http'
require 'json'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  format: 'json',
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
json = JSON.parse(res)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Response JSON Body: \#{res.body}"
puts "Response Body: \#{json['body']}"
puts "Response Original Status: \#{json['original_status']}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&format=json&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => {
    const json = JSON.parse(body);
    console.log(json.original_status);
    console.log(json.body);
  });
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&format=json&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

$json = json_decode($response);
var_dump($json['body']);
var_dump($json['original_status']);

from urllib2 import urlopen
from urllib import quote_plus
import json

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&format=json&url=' + url)

parsed_json = json.loads(handler.read())
print parsed_json['original_status']
print parsed_json['body']

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
  "encoding/json"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&format=json&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  var parsedJson map[string]interface{}
  json.Unmarshal(body, &parsedJson)

  fmt.Println("response: ", parsedJson["original_status"].(float64), parsedJson["body"].(string))
}

# pretty

• Optional
• Type boolean

If you're expecting a json response, you can optimize its readability by employing &pretty=true.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories&format=json&pretty=true"

require 'net/http'
require 'json'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  format: 'json',
  pretty: true,
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
json = JSON.parse(res)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Response JSON Body: \#{res.body}"
puts "Response Body: \#{json['body']}"
puts "Response Original Status: \#{json['original_status']}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&format=json&pretty=true&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => {
    const json = JSON.parse(body);
    console.log(json.original_status);
    console.log(json.body);
  });
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&format=json&pretty=true&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

$json = json_decode($response);
var_dump($json['body']);
var_dump($json['original_status']);

from urllib2 import urlopen
from urllib import quote_plus
import json

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&format=json&pretty=true&url=' + url)

parsed_json = json.loads(handler.read())
print parsed_json['original_status']
print parsed_json['body']

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
  "encoding/json"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&format=json&pretty=true&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  var parsedJson map[string]interface{}
  json.Unmarshal(body, &parsedJson)

  fmt.Println("response: ", parsedJson["original_status"].(float64), parsedJson["body"].(string))
}

# user_agent

• Optional
• Type string

If you want to make the request with a custom user agent, you can pass it here and our servers will forward it to the requested url.

We recommend to NOT use this parameter and let our artificial intelligence handle this.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&user_agent=Mozilla%2F5.0+%28Macintosh%3B+Intel+Mac+OS+X+10_12_5%29+AppleWebKit%2F603.2.4+%28KHTML%2C+like+Gecko%29+Version%2F10.1.1+Safari%2F603.2.4&url=https%3A%2F%2Fpostman-echo.com%2Fheaders"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15',
  url: 'https://postman-echo.com/headers'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://postman-echo.com/headers');
const userAgent = encodeURIComponent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&user_agent=' + userAgent + '&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://postman-echo.com/headers');
$userAgent = urlencode('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&user_agent=' . $userAgent . '&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://postman-echo.com/headers')
user_agent = quote_plus('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&user_agent=' + user_agent + '&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  urlString := url.QueryEscape("https://postman-echo.com/headers")
  userAgent := url.QueryEscape("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.1 Safari/605.1.15")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&user_agent=" + userAgent + "&url=" + urlString)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# page_wait

• Optional
• Type number

If you are using the JavaScript token, you can optionally pass page_wait parameter to wait an amount of milliseconds before the browser captures the resulting html code.

This is useful in cases where the page takes some seconds to render or some ajax needs to be loaded before the html is being captured.

curl "https://api.crawlbase.com/?token=_JS_TOKEN_&page_wait=1000&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_JS_TOKEN_',
  page_wait: '1000',
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_JS_TOKEN_&page_wait=1000&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_JS_TOKEN_&page_wait=1000&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_JS_TOKEN_&page_wait=1000&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_JS_TOKEN_&page_wait=1000&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# ajax_wait

• Optional
• Type boolean

If you are using the JavaScript token, you can optionally pass ajax_wait parameter to wait for the ajax requests to finish before getting the html response.

curl "https://api.crawlbase.com/?token=_JS_TOKEN_&ajax_wait=true&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_JS_TOKEN_',
  ajax_wait: true,
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_JS_TOKEN_&ajax_wait=true&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_JS_TOKEN_&ajax_wait=true&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_JS_TOKEN_&ajax_wait=true&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_JS_TOKEN_&ajax_wait=true&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# css_click_selector

• Optional
• Type string

# Single CSS Selector

If you are using the JavaScript token, you can optionally pass the css_click_selector parameter to click an element on the page before the browser captures the resulting HTML code.

This parameter accepts a fully specified and valid CSS selector. For example, you can use an ID selector such as #some-button, a class selector like .some-other-button, or an attribute selector such as [data-tab-item="tab1"]. It is important to ensure that the CSS selector is properly encoded to avoid errors.

Please note, if the selector is not found on the page, the request will fail with pc_status 595. To receive a response even when a selector is not found, you can append a universally found selector, like body, as a fallback. For example: #some-button,body.

# Multiple CSS Selectors

To accommodate scenarios where multiple elements may need to be clicked sequentially before capturing the page content, the css_click_selector parameter can now accept multiple CSS selectors. Separate each selector with a pipe (|) character. Ensure the entire value, including separators, is URL-encoded to avoid any parsing issues.

Suppose you want to click a button with the ID start-button and then a link with the class next-page-link. You would construct your css_click_selector parameter like this:

• Original selectors: #start-button|.next-page-link
• URL-encoded: %23start-button%7C.next-page-link

Append this parameter to your API request to ensure both elements are clicked in the order specified.

Please ensure all selectors provided are valid and present on the page to avoid errors. If any selector is not found, the request will adhere to the error handling specified above, failing with pc_status 595 unless a fallback selector is included.

curl "https://api.crawlbase.com/?token=_JS_TOKEN_&css_click_selector=%5Bdata-tab-item%3D%22overview%22%5D&page_wait=1000&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_JS_TOKEN_',
  page_wait: '1000',
  css_click_selector: '[data-tab-item="overview"]',
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_JS_TOKEN_&css_click_selector=%5Bdata-tab-item%3D%22overview%22%5D&page_wait=1000&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_JS_TOKEN_&css_click_selector=%5Bdata-tab-item%3D%22overview%22%5D&page_wait=1000&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_JS_TOKEN_&css_click_selector=%5Bdata-tab-item%3D%22overview%22%5D&page_wait=1000&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_JS_TOKEN_&css_click_selector=%5Bdata-tab-item%3D%22overview%22%5D&page_wait=1000&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# device

• Optional
• Type string

Optionally, if you don't want to specify a user_agent but you want to have the requests from a specific device, you can use this parameter.

There are two options available: desktop and mobile.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&device=mobile&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  device: 'mobile',
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&device=mobile&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&device=mobile&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&device=mobile&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&device=mobile&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# get_cookies

• Optional
• Type boolean

Optionally, if you need to get the cookies that the original website sets on the response, you can use the &get_cookies=true parameter.

The cookies will come back in the header (or in the json response if you use &format=json) as original_set_cookie.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&get_cookies=true&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  get_cookies: true,
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&get_cookies=true&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&get_cookies=true&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&get_cookies=true&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&get_cookies=true&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# get_headers

• Optional
• Type boolean

Optionally, if you need to get the headers that the original website sets on the response, you can use the &get_headers=true parameter.

The headers will come back in the response as original_header_name by default. When &format=json is passed, the header will come back as original_headers.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&get_headers=true&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  get_headers: true,
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&get_headers=true&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&get_headers=true&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&get_headers=true&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&get_headers=true&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# request_headers

• Optional
• Type string

Optionally, if you need to send request headers to the original website, you can use the &request_headers=EncodedRequestHeaders parameter.

Example request headers: accept-language:en-GB|accept-encoding:gzip

Example encoded: &request_headers=accept-language%3Aen-GB%7Caccept-encoding%3Agzip

Please note that not all request headers are allowed by the API. We recommend that you test the headers sent using this testing url: https://postman-echo.com/headers

If you need to send some additional headers which are not allowed by the API, please let us know the header names and we will authorize them for your token.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&request_headers=accept-language%3Aen-GB%7Caccept-encoding%3Agzip&url=https%3A%2F%2Fpostman-echo.com%2Fheaders"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  request_headers: 'accept-language:en-GB|accept-encoding:gzip',
  url: 'https://postman-echo.com/headers'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://postman-echo.com/headers');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&request_headers=accept-language%3Aen-GB%7Caccept-encoding%3Agzip&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://postman-echo.com/headers');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&request_headers=accept-language%3Aen-GB%7Caccept-encoding%3Agzip&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://postman-echo.com/headers')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&request_headers=accept-language%3Aen-GB%7Caccept-encoding%3Agzip&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://postman-echo.com/headers")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&request_headers=accept-language%3Aen-GB%7Caccept-encoding%3Agzip&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# set_cookies

• Optional
• Type string

Optionally, if you need to send cookies to the original website, you can use the &cookies=EncodedCookies parameter.

Example cookies: key1=value1; key2=value2; key3=value3

Example encoded: &cookies=key1%3Dvalue1%3B%20key2%3Dvalue2%3B%20key3%3Dvalue3

We recommend that you test the cookies sent using this testing url: https://postman-echo.com/cookies

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&cookies=key1%3Dvalue1%3B%20key2%3Dvalue2%3B%20key3%3Dvalue3&url=https%3A%2F%2Fpostman-echo.com%2Fcookies"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  cookies: 'key1=value1; key2=value2; key3=value3',
  url: 'https://postman-echo.com/cookies'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://postman-echo.com/cookies');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&cookies=key1%3Dvalue1%3B%20key2%3Dvalue2%3B%20key3%3Dvalue3&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://postman-echo.com/cookies');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&cookies=key1%3Dvalue1%3B%20key2%3Dvalue2%3B%20key3%3Dvalue3&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://postman-echo.com/cookies')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&cookies=key1%3Dvalue1%3B%20key2%3Dvalue2%3B%20key3%3Dvalue3&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://postman-echo.com/cookies")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&cookies=key1%3Dvalue1%3B%20key2%3Dvalue2%3B%20key3%3Dvalue3&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# cookies_session

• Optional
• Type string

If you need to send the cookies that come back on every request to all subsequent calls, you can use the &cookies_session= parameter.

The &cookies_session= parameter can be any value. Simply send a new value to create a new cookies session (this will allow you to send the returned cookies from the subsequent calls to the next API calls with that cookies session value). The value can be a maximum of 32-characters and sessions expire in 300 seconds after the last API call.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&cookies_session=1234abcd&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  cookies_session: '1234abcd',
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&cookies_session=1234abcd&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&cookies_session=1234abcd&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&cookies_session=1234abcd&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&cookies_session=1234abcd&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# screenshot

• Optional
• Type boolean

If you are using the JavaScript token, you can optionally pass &screenshot=true parameter to get a screenshot in the JPEG format of the whole crawled page.

Crawlbase will send you back the screenshot_url in the response headers (or in the json response if you use &format=json). The screenshot_url expires in one hour.

Note: When using the screenshot=true parameter, you can customize the screenshot output with these additional parameters:

• mode: Set to viewport to capture only the viewport instead of the full page. Default is fullpage.
• width: Specify maximum width in pixels (only works with mode=viewport). Default is screen width.
• height: Specify maximum height in pixels (only works with mode=viewport). Default is screen height.

Example: &screenshot=true&mode=viewport&width=1200&height=800

curl "https://api.crawlbase.com/?token=_JS_TOKEN_&screenshot=true&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_JS_TOKEN_',
  screenshot: true,
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"
puts "Response Screenshot URL: \#{res['screenshot_url']}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_JS_TOKEN_&screenshot=true&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_JS_TOKEN_&screenshot=true&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_JS_TOKEN_&screenshot=true&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_JS_TOKEN_&screenshot=true&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# store

• Optional
• Type boolean

Optionally pass &store=true parameter to store a copy of the API response in the Crawlbase Cloud Storage (opens new window).

Crawlbase will send you back the storage_url in the response headers (or in the json response if you use &format=json).

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&store=true&url=https%3A%2F%2Fgithub.com%2Fcrawlbase%3Ftab%3Drepositories"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  store: true,
  url: 'https://github.com/crawlbase?tab=repositories'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"
puts "Response Storage URL: \#{res['storage_url']}"

const https = require('https');

const url = encodeURIComponent('https://github.com/crawlbase?tab=repositories');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&store=true&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://github.com/crawlbase?tab=repositories');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&store=true&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://github.com/crawlbase?tab=repositories')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&store=true&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://github.com/crawlbase?tab=repositories")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&store=true&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# scraper

• Optional
• Type string

Returns back the information parsed according to the specified scraper. Check the list of all the available data scrapers (opens new window) list of all the available data scrapers to see which one to choose.

The response will come back as JSON.

Please note: Scraper is an optional parameter. If you don't use it, you will receive back the full HTML of the page so you can scrape it freely.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&scraper=amazon-product-details&url=https%3A%2F%2Fwww.amazon.com%2Fdp%2FB0B7CBZZ16"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  scraper: 'amazon-product-details',
  url: 'https://www.amazon.com/dp/B0B7CBZZ16'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://www.amazon.com/dp/B0B7CBZZ16');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&scraper=amazon-product-details&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://www.amazon.com/dp/B0B7CBZZ16');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&scraper=amazon-product-details&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://www.amazon.com/dp/B0B7CBZZ16')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&scraper=amazon-product-details&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://www.amazon.com/dp/B0B7CBZZ16")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&scraper=amazon-product-details&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# async

• Optional
• Type boolean
• Currently only linkedin.com is supported using this parameter. Talk to us if you require other domains on async mode.

Optionally pass &async=true parameter to crawl the requested URL asynchronously. Crawlbase will store the resulted page in the Crawlbase Cloud Storage (opens new window).

As a result of doing a call with async=true, Crawlbase will send you back the request identifier rid in the json response. You will need to store the RID to retrieve the document from the storage. With the RID, you can then use the Cloud Storage API (opens new window) to retrieve the resulted page.

You can use the async=true parameter in combination with other API parameter like for example &async=true&autoparse=true.

Example of request with async=true call:

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&async=true&url=https%3A%2F%2Fwww.linkedin.com%2Fcompany%2Fcrawlbase"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  async: true,
  url: 'https://www.linkedin.com/company/crawlbase'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://www.linkedin.com/company/crawlbase');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&async=true&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://www.linkedin.com/company/crawlbase');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&async=true&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://www.linkedin.com/company/crawlbase')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&async=true&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://www.linkedin.com/company/crawlbase")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&async=true&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

Example of response with async=true call:

{ "rid": "1e92e8bff32c31c2728714d4" }

# autoparse

• Optional
• Type boolean

Optionally, if you need to get the scraped data of the page that you requested, you can pass &autoparse=true parameter.

The response will come back as JSON. The structure of the response varies depending on the URL that you sent.

Please note: &autoparse=true is an optional parameter. If you don't use it, you will receive back the full HTML of the page so you can scrape it freely.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&autoparse=true&url=https%3A%2F%2Fwww.amazon.com%2Fdp%2FB0B7CBZZ16"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  autoparse: 'true',
  url: 'https://www.amazon.com/dp/B0B7CBZZ16'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://www.amazon.com/dp/B0B7CBZZ16');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&autoparse=true&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://www.amazon.com/dp/B0B7CBZZ16');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&autoparse=true&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://www.amazon.com/dp/B0B7CBZZ16')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&autoparse=true&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://www.amazon.com/dp/B0B7CBZZ16")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&autoparse=true&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# country

• Optional
• Type string

If you want your requests to be geolocated from a specific country, you can use the &country= parameter, like &country=US (two-character country code).

Please take into account that specifying a country can reduce the number of successful requests you get back, so use it wisely and only when geolocation crawls are required.

Also note that some websites like Amazon are routed via different special proxies and all countries are allowed regardless of being in the list or not.

You have access to the following countries

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&country=US&url=https%3A%2F%2Fpostman-echo.com%2Fip"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  country: 'US',
  url: 'https://postman-echo.com/ip'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://postman-echo.com/ip');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&country=US&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://postman-echo.com/ip');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&country=US&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://postman-echo.com/ip')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&country=US&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://postman-echo.com/ip")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&country=US&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# tor_network

• Optional
• Type boolean

If you want to crawl onion websites over the Tor network, you can pass the &tor_network=true parameter.

curl "https://api.crawlbase.com/?token=_USER_TOKEN_&tor_network=true&url=https%3A%2F%2Fwww.facebookcorewwwi.onion%2F"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_USER_TOKEN_',
  tor_network: 'true',
  url: 'https://www.facebookcorewwwi.onion/'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://www.facebookcorewwwi.onion/');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_USER_TOKEN_&tor_network=true&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://www.facebookcorewwwi.onion/');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_USER_TOKEN_&tor_network=true&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://www.facebookcorewwwi.onion/')

handler = urlopen('https://api.crawlbase.com/?token=_USER_TOKEN_&tor_network=true&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://www.facebookcorewwwi.onion/")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_USER_TOKEN_&tor_network=true&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}

# scroll

• Optional
• Type boolean

If you are using the JavaScript token, you can optionally pass &scroll=true to the API this will by default scroll for a scroll_interval of 10 seconds.

If you want to scroll more than 10 seconds please send the &scroll=true&scroll_interval=20. Those parameters will instruct the browser to scroll for 20 seconds after loading the page. The maximum scroll interval is 60 seconds, after 60 seconds of a scroll, the system captures the data and brings it back to you.

The default scroll interval is 10 seconds. Every 5 seconds of successful scroll counts as extra JS request on the Crawling API, so let us assume you send a scroll_interval 20, our system tries to scroll the page for a maximum of 20 seconds, if it only was able to scroll for 10 seconds, only 2 extra requests are consumed instead of 4.

Note: Please make sure to keep your connection open up to 90 seconds if you are intending to scroll for 60 seconds.

Important: Some domains require higher system timeouts which are set automatically. When combined with scroll and page_wait parameters, this may result in additional request counts. Contact support if you need to optimize these settings for specific domains.

curl "https://api.crawlbase.com/?token=_JS_TOKEN_&scroll=true&url=https%3A%2F%2Fwww.reddit.com%2Fsearch%2F%3Fq%3Dcrawlbase"

require 'net/http'

uri = URI('https://api.crawlbase.com')
uri.query = URI.encode_www_form({
  token: '_JS_TOKEN_',
  scroll: true,
  url: 'https://www.reddit.com/search/?q=crawlbase'})

res = Net::HTTP.get_response(uri)
puts "Response HTTP Status Code: \#{res.code}"
puts "Response HTTP Header Original Status: \#{res['original_status']}"
puts "Response HTTP Response Body: \#{res.body}"

const https = require('https');

const url = encodeURIComponent('https://www.reddit.com/search/?q=crawlbase');
const options = {
  hostname: 'api.crawlbase.com',
  path: '/?token=_JS_TOKEN_&scroll=true&url=' + url
};

https.request(options, (response) => {
  let body = '';
  response.on('data', chunk => body += chunk).on('end', () => console.log(body));
}).end();

$url = urlencode('https://www.reddit.com/search/?q=crawlbase');

$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.crawlbase.com/?token=_JS_TOKEN_&scroll=true&url=' . $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

$response = curl_exec($ch);
curl_close($ch);

var_dump($response);

from urllib2 import urlopen
from urllib import quote_plus

url = quote_plus('https://www.reddit.com/search/?q=crawlbase')

handler = urlopen('https://api.crawlbase.com/?token=_JS_TOKEN_&scroll=true&url=' + url)

print handler.read()

package main

import (
  "net/http"
  "fmt"
  "io/ioutil"
  "net/url"
)

func main() {
  url := url.QueryEscape("https://www.reddit.com/search/?q=crawlbase")
  resp, _ := http.Get("https://api.crawlbase.com/?token=_JS_TOKEN_&scroll=true&url=" + url)

  defer resp.Body.Close()
  body, _ := ioutil.ReadAll(resp.Body)

  fmt.Println("response Body: ", string(body))
}