GoProxies Scraping Examples

Examples showing how to use GoProxies with Python, cURL, Node.JS, Java, Ruby, PHP and Go.

GoProxies can be easily integrated into your scraping setup, no matter which programming language you decide to use. Below are demo examples for scraping amazon.com (for testing purposes) in several programming languages you can copy, paste and run right away - just make sure to replace credentials with your own API username and password.

curl --proxytunnel --proxy "https://customer-USERNAME:[email protected]:1080" https://www.amazon.com/dp/B07RZ74VLR

// How to run:
//   npm install axios https-proxy-agent
//   node proxy_example.js
//
// Save as proxy_example.js

const axios = require('axios');
const HttpsProxyAgent = require('https-proxy-agent');

const USERNAME = process.env.GOPROXIES_USER || 'customer-USERNAME';
const PASSWORD = process.env.GOPROXIES_PASS || 'PASSWORD';
const proxy = `https://${USERNAME}:${PASSWORD}@proxy.goproxies.com:1080`;
const agent = new HttpsProxyAgent(proxy);

// Test proxy (IP check)
axios.get('https://ip.goproxies.com', { httpsAgent: agent, timeout: 30000 })
  .then(res => console.log('IP check:', res.data))
  .catch(err => console.error('IP error:', err.message));

// Demo request to Amazon (for demo only)
axios.get('https://www.amazon.com/dp/B07RZ74VLR', { httpsAgent: agent, timeout: 30000 })
  .then(res => console.log('Page snippet:', res.data.slice(0,500)))
  .catch(err => console.error('Page error:', err.message));

# How to run:
#   pip install requests
#   python proxy_example.py
#
# Save as proxy_example.py

import os
import requests

# Replace or use env vars
USERNAME = os.getenv("GOPROXIES_USER", "customer-USERNAME")
PASSWORD = os.getenv("GOPROXIES_PASS", "PASSWORD")
proxy_url = f"https://{USERNAME}:{PASSWORD}@proxy.goproxies.com:1080"

proxies = {
    "http": proxy_url,
    "https": proxy_url,
}

# Test proxy (IP check)
resp = requests.get("https://ip.goproxies.com", proxies=proxies, timeout=30)
print("IP check:", resp.text)

# Demo request to Amazon (for demo only)
resp2 = requests.get("https://www.amazon.com/dp/B07RZ74VLR", proxies=proxies, timeout=30)
print("Status:", resp2.status_code)
print("Page snippet:", resp2.text[:500])

# How to run:
#   ruby proxy_example.rb
#
# Save as proxy_example.rb

require 'net/http'
require 'uri'

proxy_host = 'proxy.goproxies.com'
proxy_port = 1080
proxy_user = ENV['GOPROXIES_USER'] || 'customer-USERNAME'
proxy_pass = ENV['GOPROXIES_PASS'] || 'PASSWORD'

uri = URI('https://www.amazon.com/dp/B07RZ74VLR')

Net::HTTP.start(uri.host, uri.port,
                proxy_host, proxy_port, proxy_user, proxy_pass,
                use_ssl: uri.scheme == 'https') do |http|
  req = Net::HTTP::Get.new(uri)
  res = http.request(req)
  puts res.code
  puts res.body[0..499]  # first 500 chars
end

// How to run:
//   Requires Java 11+
//   javac ProxyExample.java
//   java ProxyExample
//
// Save as ProxyExample.java

import java.net.*;
import java.net.http.*;
import java.time.Duration;
import java.util.Base64;

public class ProxyExample {
    public static void main(String[] args) throws Exception {
        String proxyHost = "proxy.goproxies.com";
        int proxyPort = 1080;
        String user = System.getenv().getOrDefault("GOPROXIES_USER", "customer-USERNAME");
        String pass = System.getenv().getOrDefault("GOPROXIES_PASS", "PASSWORD");
        String auth = Base64.getEncoder().encodeToString((user + ":" + pass).getBytes());

        HttpClient client = HttpClient.newBuilder()
            .connectTimeout(Duration.ofSeconds(20))
            .proxy(ProxySelector.of(new InetSocketAddress(proxyHost, proxyPort)))
            .build();

        HttpRequest request = HttpRequest.newBuilder()
            .uri(URI.create("https://www.amazon.com/dp/B07RZ74VLR"))
            .timeout(Duration.ofSeconds(30))
            .header("Proxy-Authorization", "Basic " + auth) // proxy auth header
            .GET()
            .build();

        HttpResponse<String> resp = client.send(request, HttpResponse.BodyHandlers.ofString());
        System.out.println(resp.statusCode());
        System.out.println(resp.body().substring(0, Math.min(500, resp.body().length())));
    }
}

<?php
// How to run:
//   php proxy_example.php
//
// Save as proxy_example.php

$user = getenv('GOPROXIES_USER') ?: 'customer-USERNAME';
$pass = getenv('GOPROXIES_PASS') ?: 'PASSWORD';

$ch = curl_init('https://www.amazon.com/dp/B07RZ74VLR');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_PROXY, 'proxy.goproxies.com:1080');
curl_setopt($ch, CURLOPT_PROXYUSERPWD, "$user:$pass");
curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, true);
$response = curl_exec($ch);
if (curl_errno($ch)) {
    echo 'Error: ' . curl_error($ch);
} else {
    echo curl_getinfo($ch, CURLINFO_HTTP_CODE) . PHP_EOL;
    echo substr($response, 0, 500);
}
curl_close($ch);

// How to run:
//   go run main.go
//
// Save as main.go

package main

import (
    "fmt"
    "io"
    "net/http"
    "net/url"
    "os"
    "time"
)

func main() {
    user := os.Getenv("GOPROXIES_USER")
    if user == "" {
        user = "customer-USERNAME"
    }
    pass := os.Getenv("GOPROXIES_PASS")
    if pass == "" {
        pass = "PASSWORD"
    }
    proxyStr := "http://" + user + ":" + pass + "@proxy.goproxies.com:1080"
    proxyURL, _ := url.Parse(proxyStr)

    tr := &http.Transport{
        Proxy: http.ProxyURL(proxyURL),
    }
    client := &http.Client{
        Transport: tr,
        Timeout:   30 * time.Second,
    }

    resp, err := client.Get("https://www.amazon.com/dp/B07RZ74VLR")
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close()
    body, _ := io.ReadAll(resp.Body)
    fmt.Println(resp.StatusCode)
    if len(body) > 500 {
        fmt.Println(string(body[:500]))
    } else {
        fmt.Println(string(body))
    }
}

PreviousModels NextHTTP Error Response codes

Last updated 2 months ago

Was this helpful?