213 d_handle = curl_easy_init();
214 if (!d_handle)
throw BESInternalError(
"Could not allocate CURL handle", __FILE__, __LINE__);
216 curl::set_error_buffer(d_handle, d_errbuf);
218 res = curl_easy_setopt(d_handle, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
219 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_SSLVERSION", d_errbuf, __FILE__, __LINE__);
223 res = curl_easy_setopt(d_handle, CURLOPT_DEBUGFUNCTION, curl_trace);
224 curl::check_setopt_result(res, prolog,
"CURLOPT_DEBUGFUNCTION", d_errbuf, __FILE__, __LINE__);
227 res = curl_easy_setopt(d_handle, CURLOPT_VERBOSE, 1L);
228 curl::check_setopt_result(res, prolog,
"CURLOPT_VERBOSE", d_errbuf, __FILE__, __LINE__);
231 res = curl_easy_setopt(d_handle, CURLOPT_HEADERFUNCTION, chunk_header_callback);
232 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HEADERFUNCTION", d_errbuf, __FILE__, __LINE__);
235 res = curl_easy_setopt(d_handle, CURLOPT_WRITEFUNCTION, chunk_write_data);
236 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEFUNCTION", d_errbuf, __FILE__, __LINE__);
238#ifdef CURLOPT_TCP_KEEPALIVE
240 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPALIVE, 1L);
241 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPALIVE", d_errbuf, __FILE__, __LINE__);
244#ifdef CURLOPT_TCP_KEEPIDLE
246 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPIDLE, 120L);
247 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPIDLE", d_errbuf, __FILE__, __LINE__);
250#ifdef CURLOPT_TCP_KEEPINTVL
252 res = curl_easy_setopt(d_handle, CURLOPT_TCP_KEEPINTVL, 120L)
253 curl::check_setopt_result(res, prolog,
"CURLOPT_TCP_KEEPINTVL", d_errbuf, __FILE__, __LINE__);
318 string reason =
"The requested resource does not match any of the AllowedHost rules.";
321 ss <<
"ERROR! The chunk url "<< chunk->
get_data_url()->str() <<
" was rejected because: " << reason;
325 std::lock_guard<std::recursive_mutex> lock_me(d_get_easy_handle_mutex);
328 for (
auto i = d_easy_handles.begin(), e = d_easy_handles.end(); i != e; ++i) {
329 if (!(*i)->d_in_use) {
337 handle->d_in_use =
true;
340 handle->d_chunk = chunk;
342 CURLcode res = curl_easy_setopt(handle->d_handle, CURLOPT_URL, chunk->
get_data_url()->str().c_str());
343 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_URL", handle->d_errbuf, __FILE__, __LINE__);
347 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_RANGE", handle->d_errbuf, __FILE__, __LINE__);
350 res = curl_easy_setopt(handle->d_handle, CURLOPT_HEADERDATA,
reinterpret_cast<void *
>(chunk));
351 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HEADERDATA", handle->d_errbuf, __FILE__, __LINE__);
354 res = curl_easy_setopt(handle->d_handle, CURLOPT_WRITEDATA,
reinterpret_cast<void *
>(chunk));
355 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_WRITEDATA", handle->d_errbuf, __FILE__, __LINE__);
358 res = curl_easy_setopt(handle->d_handle, CURLOPT_PRIVATE,
reinterpret_cast<void *
>(handle));
359 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_PRIVATE", handle->d_errbuf, __FILE__, __LINE__);
362 res = curl_easy_setopt(handle->d_handle, CURLOPT_COOKIEFILE, curl::get_cookie_filename().c_str());
363 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_COOKIEFILE", handle->d_errbuf, __FILE__, __LINE__);
365 res = curl_easy_setopt(handle->d_handle, CURLOPT_COOKIEJAR, curl::get_cookie_filename().c_str());
366 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_COOKIEJAR", handle->d_errbuf, __FILE__, __LINE__);
369 res = curl_easy_setopt(handle->d_handle, CURLOPT_FOLLOWLOCATION, 1);
370 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_FOLLOWLOCATION", handle->d_errbuf, __FILE__, __LINE__);
372 res = curl_easy_setopt(handle->d_handle, CURLOPT_MAXREDIRS, curl::max_redirects());
373 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_MAXREDIRS", handle->d_errbuf, __FILE__, __LINE__);
376 res = curl_easy_setopt(handle->d_handle, CURLOPT_USERAGENT, curl::hyrax_user_agent().c_str());
377 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_USERAGENT", handle->d_errbuf, __FILE__, __LINE__);
382 res = curl_easy_setopt(handle->d_handle, CURLOPT_HTTPAUTH, (
long) CURLAUTH_ANY);
383 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HTTPAUTH", handle->d_errbuf, __FILE__, __LINE__);
386 res = curl_easy_setopt(handle->d_handle, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
387 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NETRC", handle->d_errbuf, __FILE__, __LINE__);
391 string netrc_file = curl::get_netrc_filename();
392 if (!netrc_file.empty()) {
393 res = curl_easy_setopt(handle->d_handle, CURLOPT_NETRC_FILE, netrc_file.c_str());
394 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_NETRC_FILE", handle->d_errbuf, __FILE__, __LINE__);
398 if (credentials && credentials->
is_s3_cred()) {
400 prolog <<
"Got AccessCredentials instance: " << endl << credentials->to_json() << endl);
403 const std::time_t request_time = std::time(0);
405 const std::string auth_header =
406 AWSV4::compute_awsv4_signature(
409 credentials->
get(AccessCredentials::ID_KEY),
410 credentials->
get(AccessCredentials::KEY_KEY),
411 credentials->
get(AccessCredentials::REGION_KEY),
415 handle->d_request_headers = curl::append_http_header((curl_slist *)0,
"Authorization", auth_header);
416 handle->d_request_headers = curl::append_http_header(handle->d_request_headers,
"x-amz-content-sha256",
417 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
418 handle->d_request_headers = curl::append_http_header(handle->d_request_headers,
"x-amz-date", AWSV4::ISO8601_date(request_time));
420 res = curl_easy_setopt(handle->d_handle, CURLOPT_HTTPHEADER, handle->d_request_headers);
421 curl::eval_curl_easy_setopt_result(res, prolog,
"CURLOPT_HTTPHEADER", handle->d_errbuf, __FILE__, __LINE__);