tonelm54
asked on
Keep getting 503 on Curl request
I have some code which downloads a Excel document from a website, the document is quite large, however other documents which are larger seem to be working ok. Just seems to be this one that is returning 503 errors when I try I download it.
What I fail to understand is my catch is never run, and I get nothing in my error_logs.
If I download the file using my browser it all works fine, just need it to authenticate with credentials, hence having to use curl.
Does anyone have any suggestions? Or anyway of looking further into the logs for what the error is?
Thank you
<?php
$source = "https://mydomain.service-now.com/myTableName.do?EXCEL";
$SN['username'] = "myUsername";
$SN['password'] = "myPassword";
$fp = fopen("test_myTableName.xls", 'w');
$ch = curl_init($source);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_ANY ) ;
curl_setopt($ch, CURLOPT_USERPWD, $SN['username'] . ":" . $SN['password']);
curl_setopt($ch, CURLOPT_FILE, $fp);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_VERBOSE, 1);
curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT ,0);
curl_setopt($ch, CURLOPT_TIMEOUT, 600);
curl_setopt($ch, CURLOPT_FAILONERROR, 0);
try {
curl_exec($ch);
echo "Successfully downloaded " . $tableID . " without error";
} catch (Exception $e) {
echo "Finished downloading " . $tableID . " with error - " . curl_error($ch);
exit();
}
?>
What I fail to understand is my catch is never run, and I get nothing in my error_logs.
If I download the file using my browser it all works fine, just need it to authenticate with credentials, hence having to use curl.
Does anyone have any suggestions? Or anyway of looking further into the logs for what the error is?
Thank you
I don't know whether cURL throws Exception. I've always tested the response with something like this code snippet. You might give this a try and show us what you get back...
<?php // /demo/curl_get_response_object_example.php
/**
* Demonstrate the basics of cURL GET-method request
*
* http://curl.haxx.se/libcurl/c/libcurl-errors.html
*/
error_reporting(E_ALL);
// USAGE EXAMPLE: BECAUSE IT IS ON MY SERVER, I HAVE HARD-CODED THIS
$url = 'https://twitter.com/RayPaseur';
// TRY THE REMOTE WEB SERVICE
$response = new GET_Response_Object($url);
// SHOW THE WORK PRODUCT
echo "<pre>";
if (!$response->document) var_dump($response); // ON FAILURE
echo htmlentities($response->document); // ON SUCCESS
// SHOW THE COOKIES, IF ANY
echo PHP_EOL;
echo file_get_contents('cookie.txt');
Class GET_Response_Object
{
public $href, $title, $http_code, $errno, $info, $document;
public function __construct($href, $user=NULL, $pass=NULL, $get_array=[], $title=NULL)
{
// ACTIVATE THIS TO AVOID TIMEOUT FOR LONG RUNNING SCRIPT
// set_time_limit(10);
// STORE THE CALL INFORMATION
$this->href = $href;
$this->title = $title;
// PREPARE THE GET STRING
$get_string = http_build_query($get_array);
if ($get_string) $get_string = '?' . $get_string;
// MAKE THE REQUEST
if (!$response = $this->my_curl($href, $user, $pass, $get_string))
{
// ACTIVATE THIS TO SEE THE ERRORS AS THEY OCCUR
trigger_error("Errno: $this->errno; HTTP: $this->http_code; URL: $this->href", E_USER_WARNING);
}
else
{
return $response;
}
}
protected function my_curl($url, $user, $pass, $get_string, $timeout=3)
{
// PREPARE THE CURL CALL
$curl = curl_init();
// HEADERS AND OPTIONS APPEAR TO BE A FIREFOX BROWSER REFERRED BY GOOGLE
$header[] = "Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5";
$header[] = "Cache-Control: max-age=0";
$header[] = "Connection: keep-alive";
$header[] = "Keep-Alive: 300";
$header[] = "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7";
$header[] = "Accept-Language: en-us,en;q=0.5";
$header[] = "Pragma: "; // BROWSERS USUALLY LEAVE THIS BLANK
// SET THE CURL OPTIONS - SEE http://php.net/manual/en/function.curl-setopt.php
curl_setopt( $curl, CURLOPT_URL, $url . $get_string );
curl_setopt( $curl, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; rv:49.0) Gecko/20100101 Firefox/49.0' );
curl_setopt( $curl, CURLOPT_HTTPHEADER, $header );
curl_setopt( $curl, CURLOPT_REFERER, 'https://www.google.com' );
curl_setopt( $curl, CURLOPT_ENCODING, 'gzip,deflate' );
curl_setopt( $curl, CURLOPT_AUTOREFERER, TRUE );
curl_setopt( $curl, CURLOPT_RETURNTRANSFER, TRUE );
curl_setopt( $curl, CURLOPT_FOLLOWLOCATION, TRUE );
curl_setopt( $curl, CURLOPT_TIMEOUT, $timeout );
curl_setopt( $curl, CURLOPT_HTTPAUTH, CURLAUTH_ANY );
curl_setopt( $curl, CURLOPT_USERPWD, "$user:$pass" );
// GET GOOD ERROR MESSAGES
curl_setopt( $curl, CURLOPT_VERBOSE, TRUE );
curl_setopt( $curl, CURLOPT_FAILONERROR, TRUE );
// IF USING SSL, THIS INFORMATION MAY BE IMPORTANT
// http://php.net/manual/en/function.curl-setopt.php#110457
// http://php.net/manual/en/function.curl-setopt.php#115993
// http://php.net/manual/en/function.curl-setopt.php#113754
// REDACTED IN 2015 curl_setopt( $curl, CURLOPT_SSLVERSION, 3 );
curl_setopt( $curl, CURLOPT_SSL_VERIFYHOST, FALSE );
curl_setopt( $curl, CURLOPT_SSL_VERIFYPEER, FALSE );
// SET THE LOCATION OF THE COOKIE JAR (THIS FILE WILL BE OVERWRITTEN)
curl_setopt( $curl, CURLOPT_COOKIEFILE, 'cookie.txt' );
curl_setopt( $curl, CURLOPT_COOKIEJAR, 'cookie.txt' );
// RUN THE CURL REQUEST AND GET THE RESULTS
$this->document = curl_exec($curl);
$this->errno = curl_errno($curl);
$this->info = curl_getinfo($curl);
$this->http_code = $this->info['http_code'];
curl_close($curl);
return $this;
}
}
ASKER
Interestingly enough I have the same issue with using file_put_contents:-
Works fine still on my machine though :-S
set_time_limit(0);
file_put_contents("test-dwn.xls",file_get_contents("https://myUsername:myPassword@myInstance.service-now.com/myTable.do?EXCEL"));
Works fine still on my machine though :-S
ASKER CERTIFIED SOLUTION
membership
This solution is only available to members.
To access this solution, you must be a member of Experts Exchange.
Agree with @gr8gonzo about CURLOPT_SSL_VERIFYHOST and CURLOPT_SSL_VERIFYPEER. Setting these to FALSE is an expedient for one-time runs, not something to use in a deployed application.
If your PHP installation doesn't have an up-to-date CA root certificate bundle, consider downloading the one at the curl website.
Check your cURL version; you want to be using using cURL 7.34 or newer.
Were you able to test the "get_response_object" code sample posted above? We have to rely on you to test and get the results back to us; we do not have your credentials to make the tests.
If your PHP installation doesn't have an up-to-date CA root certificate bundle, consider downloading the one at the curl website.
Check your cURL version; you want to be using using cURL 7.34 or newer.
Were you able to test the "get_response_object" code sample posted above? We have to rely on you to test and get the results back to us; we do not have your credentials to make the tests.
ASKER
Weirdly enough this just started to work, after multiple moans to my hosting provider :-S
Just retry it. Otherwise ask the administrator of that machine, why the service is not available.