30 {
"$Id: HTTPConnect.cc 25101 2011-12-19 22:03:29Z jimg $"
83 #define CLIENT_ERR_MIN 400
84 #define CLIENT_ERR_MAX 417
88 "Unauthorized: Contact the server administrator.",
90 "Forbidden: Contact the server administrator.",
91 "Not Found: The data source or server could not be found.\n\
92 Often this means that the OPeNDAP server is missing or needs attention;\n\
93 Please contact the server administrator.",
94 "Method Not Allowed.",
96 "Proxy Authentication Required.",
101 "Precondition Failed.",
102 "Request Entity Too Large.",
103 "Request URI Too Large.",
104 "Unsupported Media Type.",
105 "Requested Range Not Satisfiable.",
106 "Expectation Failed."
109 #define SERVER_ERR_MIN 500
110 #define SERVER_ERR_MAX 505
113 "Internal Server Error.",
116 "Service Unavailable.",
118 "HTTP Version Not Supported."
124 http_status_to_string(
int status)
131 return string(
"Unknown Error: This indicates a problem with libdap++.\nPlease report this to support@opendap.org.");
138 class ParseHeader :
public unary_function<const string &, void>
146 ParseHeader() : type(
unknown_type), server(
"dods/0.0"), protocol(
"2.0")
149 void operator()(
const string &line)
153 if (name ==
"content-description") {
154 DBG2(cerr << name <<
": " << value << endl);
160 else if (name ==
"xdods-server" && server ==
"dods/0.0") {
161 DBG2(cerr << name <<
": " << value << endl);
164 else if (name ==
"xopendap-server") {
165 DBG2(cerr << name <<
": " << value << endl);
168 else if (name ==
"xdap") {
169 DBG2(cerr << name <<
": " << value << endl);
172 else if (server ==
"dods/0.0" && name ==
"server") {
173 DBG2(cerr << name <<
": " << value << endl);
176 else if (name ==
"location") {
177 DBG2(cerr << name <<
": " << value << endl);
181 && line.find(
"text/html") != string::npos) {
182 DBG2(cerr << name <<
": text/html..." << endl);
197 string get_protocol()
202 string get_location() {
224 save_raw_http_headers(
void *ptr,
size_t size,
size_t nmemb,
void *resp_hdrs)
226 DBG2(cerr <<
"Inside the header parser." << endl);
227 vector<string> *hdrs =
static_cast<vector<string> *
>(resp_hdrs);
230 string complete_line;
231 if (nmemb > 1 && *(static_cast<char*>(ptr) + size * (nmemb - 2)) ==
'\r')
232 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 2));
234 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 1));
237 if (complete_line !=
"" && complete_line.find(
"HTTP") == string::npos) {
238 DBG(cerr <<
"Header line: " << complete_line << endl);
239 hdrs->push_back(complete_line);
247 curl_debug(CURL *, curl_infotype info,
char *msg,
size_t size,
void *)
249 string message(msg, size);
253 cerr <<
"Text: " << message;
break;
254 case CURLINFO_HEADER_IN:
255 cerr <<
"Header in: " << message;
break;
256 case CURLINFO_HEADER_OUT:
257 cerr <<
"Header out: " << message;
break;
258 case CURLINFO_DATA_IN:
259 cerr <<
"Data in: " << message;
break;
260 case CURLINFO_DATA_OUT:
261 cerr <<
"Data out: " << message;
break;
263 cerr <<
"End: " << message;
break;
264 #ifdef CURLINFO_SSL_DATA_IN
265 case CURLINFO_SSL_DATA_IN:
266 cerr <<
"SSL Data in: " << message;
break;
268 #ifdef CURLINFO_SSL_DATA_OUT
269 case CURLINFO_SSL_DATA_OUT:
270 cerr <<
"SSL Data out: " << message;
break;
273 cerr <<
"Curl info: " << message;
break;
282 HTTPConnect::www_lib_init()
284 d_curl = curl_easy_init();
286 throw InternalErr(__FILE__, __LINE__,
"Could not initialize libcurl.");
292 if (!d_rcr->get_proxy_server_host().empty()) {
293 DBG(cerr <<
"Setting up a proxy server." << endl);
294 DBG(cerr <<
"Proxy host: " << d_rcr->get_proxy_server_host()
296 DBG(cerr <<
"Proxy port: " << d_rcr->get_proxy_server_port()
298 DBG(cerr <<
"Proxy pwd : " << d_rcr->get_proxy_server_userpw()
300 curl_easy_setopt(d_curl, CURLOPT_PROXY,
301 d_rcr->get_proxy_server_host().c_str());
302 curl_easy_setopt(d_curl, CURLOPT_PROXYPORT,
303 d_rcr->get_proxy_server_port());
306 #ifdef CURLOPT_PROXYAUTH
307 curl_easy_setopt(d_curl, CURLOPT_PROXYAUTH, (
long)CURLAUTH_ANY);
311 if (!d_rcr->get_proxy_server_userpw().empty())
312 curl_easy_setopt(d_curl, CURLOPT_PROXYUSERPWD,
313 d_rcr->get_proxy_server_userpw().c_str());
316 curl_easy_setopt(d_curl, CURLOPT_ERRORBUFFER, d_error_buffer);
319 curl_easy_setopt(d_curl, CURLOPT_FAILONERROR, 0);
324 curl_easy_setopt(d_curl, CURLOPT_HTTPAUTH, (
long)CURLAUTH_ANY);
326 curl_easy_setopt(d_curl, CURLOPT_NOPROGRESS, 1);
327 curl_easy_setopt(d_curl, CURLOPT_NOSIGNAL, 1);
328 curl_easy_setopt(d_curl, CURLOPT_HEADERFUNCTION, save_raw_http_headers);
333 curl_easy_setopt(d_curl, CURLOPT_FOLLOWLOCATION, 1);
334 curl_easy_setopt(d_curl, CURLOPT_MAXREDIRS, 5);
337 if (!d_rcr->get_validate_ssl() == 0) {
338 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, 0);
339 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, 0);
346 if (!d_cookie_jar.empty()) {
347 DBG(cerr <<
"Setting the cookie jar to: " << d_cookie_jar << endl);
348 curl_easy_setopt(d_curl, CURLOPT_COOKIEJAR, d_cookie_jar.c_str());
349 curl_easy_setopt(d_curl, CURLOPT_COOKIESESSION, 1);
353 cerr <<
"Curl version: " << curl_version() << endl;
354 curl_easy_setopt(d_curl, CURLOPT_VERBOSE, 1);
355 curl_easy_setopt(d_curl, CURLOPT_DEBUGFUNCTION, curl_debug);
362 class BuildHeaders :
public unary_function<const string &, void>
364 struct curl_slist *d_cl;
367 BuildHeaders() : d_cl(0)
370 void operator()(
const string &header)
372 DBG(cerr <<
"Adding '" << header.c_str() <<
"' to the header list."
374 d_cl = curl_slist_append(d_cl, header.c_str());
377 struct curl_slist *get_headers()
398 HTTPConnect::read_url(
const string &url, FILE *stream,
399 vector<string> *resp_hdrs,
400 const vector<string> *headers)
402 curl_easy_setopt(d_curl, CURLOPT_URL, url.c_str());
412 curl_easy_setopt(d_curl, CURLOPT_FILE, stream);
413 curl_easy_setopt(d_curl, CURLOPT_WRITEFUNCTION, &fwrite);
415 curl_easy_setopt(d_curl, CURLOPT_FILE, stream);
418 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
419 ostream_iterator<string>(cerr,
"\n")));
421 BuildHeaders req_hdrs;
422 req_hdrs = for_each(d_request_headers.begin(), d_request_headers.end(),
425 req_hdrs = for_each(headers->begin(), headers->end(), req_hdrs);
426 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, req_hdrs.get_headers());
429 bool temporary_proxy =
false;
430 if ((temporary_proxy = url_uses_no_proxy_for(url))) {
431 DBG(cerr <<
"Suppress proxy for url: " << url << endl);
432 curl_easy_setopt(d_curl, CURLOPT_PROXY, 0);
435 string::size_type at_sign = url.find(
'@');
439 if (at_sign != url.npos)
440 d_upstring = url.substr(7, at_sign - 7);
442 if (!d_upstring.empty())
443 curl_easy_setopt(d_curl, CURLOPT_USERPWD, d_upstring.c_str());
448 curl_easy_setopt(d_curl, CURLOPT_WRITEHEADER, resp_hdrs);
450 CURLcode res = curl_easy_perform(d_curl);
453 curl_slist_free_all(req_hdrs.get_headers());
454 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, 0);
457 if (temporary_proxy && !d_rcr->get_proxy_server_host().empty())
458 curl_easy_setopt(d_curl, CURLOPT_PROXY,
459 d_rcr->get_proxy_server_host().c_str());
462 throw Error(d_error_buffer);
465 res = curl_easy_getinfo(d_curl, CURLINFO_HTTP_CODE, &status);
467 throw Error(d_error_buffer);
476 HTTPConnect::url_uses_proxy_for(
const string &url)
throw()
478 if (d_rcr->is_proxy_for_used()) {
479 Regex host_regex(d_rcr->get_proxy_for_regexp().c_str());
480 int index = 0, matchlen;
481 return host_regex.search(url.c_str(), url.size(), matchlen, index) != -1;
491 HTTPConnect::url_uses_no_proxy_for(
const string &url)
throw()
493 return d_rcr->is_no_proxy_for_used()
494 && url.find(d_rcr->get_no_proxy_for_host()) != string::npos;
505 HTTPConnect::HTTPConnect(
RCReader *rcr) : d_username(
""), d_password(
""),
507 d_dap_client_protocol_major(2),
508 d_dap_client_protocol_minor(0)
518 d_request_headers.push_back(
string(
"Pragma:"));
519 string user_agent = string(
"User-Agent: ") + string(
CNAME)
520 + string(
"/") + string(
CVER);
521 d_request_headers.push_back(user_agent);
522 if (d_accept_deflate)
523 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
532 DBG2(cerr <<
"Cache object created (" << hex << d_http_cache << dec
551 DBG2(cerr <<
"Entering the HTTPConnect dtor" << endl);
553 curl_easy_cleanup(d_curl);
555 DBG2(cerr <<
"Leaving the HTTPConnect dtor" << endl);
574 cout <<
"GET " << url <<
" HTTP/1.0" << endl;
580 stream = caching_fetch_url(url);
583 stream = plain_fetch_url(url);
588 ss <<
"HTTP/1.0 " << stream->
get_status() <<
" -" << endl;
589 for (
size_t i = 0; i < stream->
get_headers()->size(); i++) {
601 cout << endl << endl;
605 if (parser.get_location() !=
"" &&
606 url.substr(0,url.find(
"?",0)).compare(parser.get_location().substr(0,url.find(
"?",0))) != 0) {
611 stream->
set_type(parser.get_object_type());
629 get_tempfile_template(
const string &file_template)
636 Regex directory(
"[-a-zA-Z0-9_:\\]*");
641 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
642 goto valid_temp_directory;
645 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
646 goto valid_temp_directory;
651 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
652 goto valid_temp_directory;
654 #else // Unix/Linux/OSX has another...
656 Regex directory(
"[-a-zA-Z0-9_/]*");
658 c = getenv(
"TMPDIR");
659 if (directory.match(c.c_str(), c.length()) && (access(c.c_str(), W_OK | R_OK) == 0))
660 goto valid_temp_directory;
665 if (access(P_tmpdir, W_OK | R_OK) == 0) {
667 goto valid_temp_directory;
673 if (directory.match(c.c_str(), c.length()) && (access(c.c_str(), W_OK | R_OK) == 0))
674 goto valid_temp_directory;
681 valid_temp_directory:
684 c +=
"\\" + file_template;
686 c +=
"/" + file_template;
713 string dods_temp = get_tempfile_template((
string)
"dodsXXXXXX");
715 vector<char> pathname(dods_temp.length() + 1);
717 strncpy(&pathname[0], dods_temp.c_str(), dods_temp.length());
719 DBG(cerr <<
"pathanme: " << &pathname[0] <<
" (" << dods_temp.length() + 1 <<
")" << endl);
722 #if defined(WIN32) || defined(TEST_WIN32_TEMPS)
723 stream = fopen(_mktemp(&pathname[0]),
"w+b");
727 stream = fdopen(mkstemp(&pathname[0]),
"w+");
732 "Failed to open a temporary file for the data values ("
736 dods_temp = &pathname[0];
748 res = unlink(name.c_str());
775 HTTPConnect::caching_fetch_url(
const string &url)
777 DBG(cerr <<
"Is this URL (" << url <<
") in the cache?... ");
779 vector<string> *headers =
new vector<string>;
784 DBGN(cerr <<
"no; getting response and caching." << endl);
785 delete headers; headers = 0;
786 time_t now = time(0);
787 HTTPResponse *rs = plain_fetch_url(url);
788 d_http_cache->
cache_response(url, now, *(rs->get_headers()), rs->get_stream());
793 DBGN(cerr <<
"yes... ");
796 DBGN(cerr <<
"and it's valid; using cached response." << endl);
797 HTTPCacheResponse *crs =
new HTTPCacheResponse(s, 200, headers, file_name, d_http_cache);
801 DBGN(cerr <<
"but it's not valid; validating... ");
808 time_t now = time(0);
812 http_status = read_url(url, body, headers, &cond_hdrs);
821 switch (http_status) {
823 DBGN(cerr <<
"read a new response; caching." << endl);
826 HTTPResponse *rs =
new HTTPResponse(body, http_status, headers, dods_temp);
832 DBGN(cerr <<
"cached response valid; updating." << endl);
838 HTTPCacheResponse *crs =
new HTTPCacheResponse(hs, 304, headers, file_name, d_http_cache);
844 if (http_status >= 400) {
845 delete headers; headers = 0;
846 string msg =
"Error while reading the URL: ";
849 +=
".\nThe OPeNDAP server returned the following message:\n";
850 msg += http_status_to_string(http_status);
854 delete headers; headers = 0;
855 throw InternalErr(__FILE__, __LINE__,
856 "Bad response from the HTTP server: " +
long_to_string(http_status));
863 throw InternalErr(__FILE__, __LINE__,
"Should never get here");
878 HTTPConnect::plain_fetch_url(
const string &url)
880 DBG(cerr <<
"Getting URL: " << url << endl);
883 vector<string> *resp_hdrs =
new vector<string>;
887 status = read_url(url, stream, resp_hdrs);
890 string msg =
"Error while reading the URL: ";
892 msg +=
".\nThe OPeNDAP server returned the following message:\n";
893 msg += http_status_to_string(status);
906 return new HTTPResponse(stream, status, resp_hdrs, dods_temp);
925 if (d_accept_deflate) {
926 if (find(d_request_headers.begin(), d_request_headers.end(),
927 "Accept-Encoding: deflate, gzip, compress") == d_request_headers.end())
928 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
929 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
930 ostream_iterator<string>(cerr,
"\n")));
933 vector<string>::iterator i;
934 i = remove_if(d_request_headers.begin(), d_request_headers.end(),
935 bind2nd(equal_to<string>(),
936 string(
"Accept-Encoding: deflate, gzip, compress")));
937 d_request_headers.erase(i, d_request_headers.end());
942 class HeaderMatch :
public unary_function<const string &, bool> {
943 const string &d_header;
945 HeaderMatch(
const string &header) : d_header(header) {}
946 bool operator()(
const string &arg) {
return arg.find(d_header) == 0; }
961 vector<string>::iterator i;
962 i = find_if(d_request_headers.begin(), d_request_headers.end(),
963 HeaderMatch(
"XDAP-Accept:"));
964 if (i != d_request_headers.end())
965 d_request_headers.erase(i);
968 d_dap_client_protocol_major = major;
969 d_dap_client_protocol_minor = minor;
970 ostringstream xdap_accept;
971 xdap_accept <<
"XDAP-Accept: " << major <<
"." << minor;
973 d_request_headers.push_back(xdap_accept.str());
975 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
976 ostream_iterator<string>(cerr,
"\n")));
1004 d_upstring = u +
":" + p;
vector< string > get_conditional_request_headers(const string &url)
bool is_cache_enabled() const
virtual int get_status() const
void set_cache_enabled(bool mode)
bool is_url_valid(const string &url)
void set_credentials(const string &u, const string &p)
static HTTPCache * instance(const string &cache_root, bool force=false)
void set_max_size(unsigned long size)
int get_ignore_expires() const
virtual void set_type(ObjectType o)
FILE * get_cached_response(const string &url, vector< string > &headers, string &cacheName)
string get_cookie_jar() const
ObjectType
The type of object in the stream coming from the data server.
HTTPResponse * fetch_url(const string &url)
virtual void set_version(const string &v)
int get_default_expires() const
A class for software fault reporting.
void parse_mime_header(const string &header, string &name, string &value)
virtual void set_protocol(const string &p)
unsigned int get_max_cached_obj() const
int get_max_cache_size() const
ObjectType get_description_type(const string &value)
void update_response(const string &url, time_t request_time, const vector< string > &headers)
void close_temp(FILE *s, const string &name)
string get_temp_file(FILE *&stream)
bool cache_response(const string &url, time_t request_time, const vector< string > &headers, const FILE *body)
void set_accept_deflate(bool defalte)
string long_to_string(long val, int base)
void set_always_validate(bool validate)
void set_xdap_protocol(int major, int minor)
void set_default_expiration(int exp_time)
bool get_use_cache() const
int get_always_validate() const
virtual vector< string > * get_headers() const
void release_cached_response(FILE *response)
void set_expire_ignored(bool mode)
string get_dods_cache_root() const
void set_max_entry_size(unsigned long size)