Zyte Support Center

Asynchronous HTTP Request with HttpClient

using System;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;

namespace AsyncProxyRequest
{
    class MainClass
    {
        static async Task Main()
        {
            string requestUri = "http://httpbin.scrapinghub.com/headers";
            string crawleraApiKey = "<CRAWLERA_APIKEY>";
            string crawleraAuth = Convert.ToBase64String(
                Encoding.Default.GetBytes(crawleraApiKey + ":"));
            string crawleraUri = string.Format(
                "{0}:{1}", "proxy.zyte.com", "8011");

            WebProxy proxy = new WebProxy(crawleraUri, true);
            CookieContainer cookieContainer = new CookieContainer();
            HttpClientHandler httpClientHandler = new HttpClientHandler()
            {
                Proxy = proxy,
                PreAuthenticate = false,
                UseDefaultCredentials = false,
                UseProxy = true,
                UseCookies = true,
                CookieContainer = cookieContainer
            };
            HttpClient client = new HttpClient(httpClientHandler);

            int _TimeoutSec = 300;
            client.Timeout = new TimeSpan(0, 0, _TimeoutSec);
            client.DefaultRequestHeaders.Add("Proxy-Authorization", "Basic " + crawleraAuth);

            try
            {
                HttpResponseMessage response = await client.GetAsync();
                HttpContent content = response.Content;
                Console.WriteLine("Response Status Code: " + (int)response.StatusCode);

                string scrapedPage = await content.ReadAsStringAsync();
                Console.WriteLine("Response Content: " + scrapedPage);
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
        }
    }
}

HTTPS Request with WebRequest

using System;
using System.IO;
using System.Net;

namespace ProxyRequest
{
    class MainClass
    {
        public static void Main (string[] args)
        {
            var myProxy = new WebProxy("http://proxy.zyte.com:8011");
            string apiKey = "<CRAWLERA_APIKEY>:"; // Note the ":" sign at the end
            var encodedApiKey = Base64Encode(apiKey);

            var request = (HttpWebRequest)WebRequest.Create("https://httpbin.org/ip");
            request.Headers.Add("Proxy-Authorization", "Basic " + encodedApiKey);
            request.Proxy = myProxy;
            request.PreAuthenticate = true;
            request.AllowAutoRedirect = false;
            request.ServerCertificateValidationCallback += (sender, certificate, chain, sslPolicyErrors) => true;
            
            var response = request.GetResponse();
            Console.WriteLine("Response Status: " + ((HttpWebResponse)response).StatusDescription);
            Console.WriteLine("\nResponse Headers:\n" + ((HttpWebResponse)response).Headers);
            var dataStream = response.GetResponseStream();
            var reader = new StreamReader(dataStream);
            string responseFromServer = reader.ReadToEnd();
            Console.WriteLine("Response Body:\n" + responseFromServer);
            reader.Close();
            response.Close();
        }
        
        public static string Base64Encode(string apiKey)
        {
            var plainTextBytes = System.Text.Encoding.UTF8.GetBytes(apiKey); 
            return System.Convert.ToBase64String(plainTextBytes);
        }
    }
}

Warning

Some HTTP client libraries including Apache HttpComponents Client and .NET don't send authentication headers by default. This can result in doubled requests so pre-emptive authentication should be enabled where this is the case.

In the above example we are making HTTPS request through Zyte Smart Proxy Manager(formerly Crawlera) . It is assumed that Smart Proxy Manager Certificate has been installed, since CONNECT method will be employed.

If you use WebClient and receive 407s from Smart Proxy Manager, try setting AllowAutoRedirect to false.