Our friend Chris suggested this new approach for using Crawlera with C#:
Hi, quick suggestion.
The code provided here is quite outdated using HttpWebRequest from .NET 1.1 introduced back in 2003. More modern C# code is more easily understood and can be more performant using async methods and the utility provided by a class such as HttpClient.
I have extended HttpClient to implement a Crawlera client, which you may find helpful to list, as in my experience few people need to use HttpWebRequest these days, and with extending HttpClient it is much more concise and easier to understand what is going on.
I also noticed that the link to the C# examples is broken from your examples page, and can only be found by going via examples in other languages.
public async Task Main(string[] args) { string apiKey = "SOME_API_KEY"; using (var client = new CrawleraClient(apiKey)) { HttpResponseMessage response = await client.GetAsync("https://httpbin.org/ip"); } }
public class CrawleraClient : HttpClient { public string ApiKey { get; set; } public static WebProxy Proxy = new WebProxy("http://proxy.crawlera.com:8010");
Our friend Chris suggested this new approach for using Crawlera with C#:
Hi, quick suggestion.
The code provided here is quite outdated using HttpWebRequest from .NET 1.1 introduced back in 2003. More modern C# code is more easily understood and can be more performant using async methods and the utility provided by a class such as HttpClient.
I have extended HttpClient to implement a Crawlera client, which you may find helpful to list, as in my experience few people need to use HttpWebRequest these days, and with extending HttpClient it is much more concise and easier to understand what is going on.
I also noticed that the link to the C# examples is broken from your examples page, and can only be found by going via examples in other languages.
public async Task Main(string[] args)
{
string apiKey = "SOME_API_KEY";
using (var client = new CrawleraClient(apiKey))
{
HttpResponseMessage response = await client.GetAsync("https://httpbin.org/ip");
}
}
public class CrawleraClient : HttpClient
{
public string ApiKey { get; set; }
public static WebProxy Proxy = new WebProxy("http://proxy.crawlera.com:8010");
public CrawleraClient(string apiKey)
: base(new HttpClientHandler() { Proxy = Proxy, PreAuthenticate = true })
{
ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true;
ApiKey = apiKey + ":";
DefaultRequestHeaders.ProxyAuthorization = new AuthenticationHeaderValue("Basic", Base64Encode(ApiKey));
}
public static string Base64Encode(string apiKey)
{
var plainTextBytes = Encoding.UTF8.GetBytes(apiKey);
return Convert.ToBase64String(plainTextBytes);
}
}
Thanks
Chris
1 Votes
7 Comments
crapo posted almost 6 years ago
Argh, need to do another edit so might as well just post it all
using System;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Security.Cryptography.X509Certificates;
namespace CrawleraClientExample
{
public class Program
{
static void Main(string[] args)
{
string apiKey = "<my Api Key here>";
var cert = new X509Certificate();
cert.Import(System.IO.Directory.GetCurrentDirectory() + "\\crawlera-ca.crt");
using (var client = new CrawleraClient(apiKey, cert))
{
var response = client.GetAsync("https://httpbin.org/ip").Result;
Console.WriteLine($"Request finished with status code: {response.StatusCode}");
Console.ReadLine();
}
}
}
public class CrawleraClient : HttpClient
{
public string ApiKey { get; set; }
public static WebProxy Proxy = new WebProxy("http://proxy.crawlera.com:8010");
public CrawleraClient(string apiKey, X509Certificate clientCertificate)
: base(CreateHttpClientHandler(clientCertificate))
{
ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true;
ApiKey = apiKey + ":";
DefaultRequestHeaders.ProxyAuthorization = new AuthenticationHeaderValue("Basic", Base64Encode(ApiKey));
}
public static HttpClientHandler CreateHttpClientHandler(X509Certificate clientCertificate)
{
var handler = new HttpClientHandler () { Proxy = Proxy, PreAuthenticate = true };
if (clientCertificate != null)
{
handler.ClientCertificateOptions = ClientCertificateOption.Manual;
handler.ClientCertificates.Add(clientCertificate);
}
return handler;
}
public static string Base64Encode(string apiKey)
{
var plainTextBytes = Encoding.UTF8.GetBytes(apiKey);
return Convert.ToBase64String(plainTextBytes);
}
}
}
0 Votes
crapo posted almost 6 years ago
Can't seem to edit, but need to include:
using System.Security.Cryptography.X509Certificates;
0 Votes
crapo posted almost 6 years ago
If you need to make an HTTPS request this is how I did it because it took me longer than it should have to work it out!
using System;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
namespace CrawleraClientExample
{
public class Program
{
static void Main(string[] args)
{
string apiKey = "<my Api Key here>";
var cert = new X509Certificate();
cert.Import(System.IO.Directory.GetCurrentDirectory() + "\\crawlera-ca.crt");
using (var client = new CrawleraClient(apiKey))
{
var response = client.GetAsync("https://httpbin.org/ip").Result;
Console.WriteLine($"Request finished with status code: {response.StatusCode}");
Console.ReadLine();
}
}
}
public class CrawleraClient : HttpClient
{
public string ApiKey { get; set; }
public static WebProxy Proxy = new WebProxy("http://proxy.crawlera.com:8010");
public class CrawleraClient : HttpClient
{
public string ApiKey { get; set; }
public static WebProxy Proxy = new WebProxy("http://proxy.crawlera.com:8010");
public CrawleraClient(string apiKey, X509Certificate clientCertificate)
: base(CreateHttpClientHandler(clientCertificate))
{
ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true;
ApiKey = apiKey + ":";
DefaultRequestHeaders.ProxyAuthorization = new AuthenticationHeaderValue("Basic", Base64Encode(ApiKey));
}
private static HttpClientHandler CreateHttpClientHandler(X509Certificate clientCertificate)
{
var handler = new HttpClientHandler() { Proxy = Proxy, PreAuthenticate = true };
if (clientCertificate != null)
{
handler.ClientCertificateOptions = ClientCertificateOption.Manual;
handler.ClientCertificates.Add(clientCertificate);
}
return handler;
}
public static string Base64Encode(string apiKey)
{
var plainTextBytes = Encoding.UTF8.GetBytes(apiKey);
return Convert.ToBase64String(plainTextBytes);
}
}
}
0 Votes
Vj posted over 6 years ago
Please help, I can't able to get any results using this code and its keep erroring
0 Votes
vaz posted about 7 years ago
<Chris response>
Hi
You need to make sure you have referenced System.Net.Http.dll correctly, then it will work; the reference is missing.
This may not work if you are using a very old version of .NET framework – but if you are running .NET framework 4.5 or above it should be fine.
Chris
0 Votes
vaz posted about 7 years ago
<Chris response>
Hi Pablo
No problem, it may be because the client is async and your main method is not async.
Here’s a full example which can be used in a Visual Studio console application project. It will write the response code to the console.
An assembly reference needs to be added for System.Net.Http. I think nothing else is needed more than the default references.
using System;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
namespace CrawleraClientExample
{
public class Program
{
static void Main(string[] args)
{
string apiKey = "<my Api Key here>";
using (var client = new CrawleraClient(apiKey))
{
var response = client.GetAsync("https://httpbin.org/ip").Result;
Console.WriteLine($"Request finished with status code: {response.StatusCode}");
Console.ReadLine();
}
}
}
public class CrawleraClient : HttpClient
{
public string ApiKey { get; set; }
public static WebProxy Proxy = new WebProxy("http://proxy.crawlera.com:8010");
public CrawleraClient(string apiKey)
: base(new HttpClientHandler() { Proxy = Proxy, PreAuthenticate = true })
{
ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true;
ApiKey = apiKey + ":";
DefaultRequestHeaders.ProxyAuthorization = new AuthenticationHeaderValue("Basic", Base64Encode(ApiKey));
}
public static string Base64Encode(string apiKey)
{
var plainTextBytes = Encoding.UTF8.GetBytes(apiKey);
return Convert.ToBase64String(plainTextBytes);
}
}
}
0 Votes
vaz posted about 7 years ago
I'm trying to test your script:
using System;
using System.IO;
using System.Net;
using System.Threading.Tasks;
namespace ProxyRequest
{
class MainClass
{
public async Task Main(string[] args)
{
string apiKey = "<my Api Key here>";
using (var client = new CrawleraClient(apiKey))
{
HttpResponseMessage response = await client.GetAsync("https://httpbin.org/ip");
}
}
public class CrawleraClient : HttpClient
{
public string ApiKey { get; set; }
public static WebProxy Proxy = new WebProxy("http://proxy.crawlera.com:8010");
public CrawleraClient(string apiKey)
: base(new HttpClientHandler() { Proxy = Proxy, PreAuthenticate = true })
{
ServicePointManager.ServerCertificateValidationCallback += (sender, cert, chain, sslPolicyErrors) => true;
ApiKey = apiKey + ":";
DefaultRequestHeaders.ProxyAuthorization = new AuthenticationHeaderValue("Basic", Base64Encode(ApiKey));
}
public static string Base64Encode(string apiKey)
{
var plainTextBytes = Encoding.UTF8.GetBytes(apiKey);
return Convert.ToBase64String(plainTextBytes);
}
}
}
}
But I'm having this error:
crawlera_c2.cs(19,35): error CS0246: The type or namespace name `HttpClient' could not be found. Are you missing an assembly reference?
I've also tried adding:
using System.Net.Http;
But still the error, even more issues appear.Do you have any suggestion?
Thanks,
Pablo Vaz
Scrapinghub Support Engineer
0 Votes
Login to post a comment