From 4e241d89207d5bc11658b291c3395d901a133aed Mon Sep 17 00:00:00 2001 From: Nelson Benitez Leon Date: Thu, 15 Mar 2012 10:52:02 +0100 Subject: [PATCH 1/5] http: try http_proxy env var when http.proxy config option is not set cURL already reads it, but if $http_proxy has username but no password cURL will not ask you for the password and so failed to authenticate returning a 407 error code. So we read it ourselves to detect that and ask for the password. Also we read it prior to connection to be able to make a proactive authentication in case the flag http_proactive_auth is set. We also take care to read env proxy var according to protocol being used in the destination url, e.g. when the url to retrieve is a https one, then the proxy env var we look at is https_proxy. To make this possible we now passed destination url parameter to get_active_slot() and get_curl_handle() functions. We also read no_proxy env var so to ignore aforementioned proxy env var if no_proxy contains an asterisk ('*') or contains the host used in url destination. Signed-off-by: Nelson Benitez Leon Signed-off-by: Junio C Hamano --- http-push.c | 24 ++++++++++++------------ http-walker.c | 2 +- http.c | 37 ++++++++++++++++++++++++++++++------- http.h | 2 +- remote-curl.c | 4 ++-- 5 files changed, 46 insertions(+), 23 deletions(-) diff --git a/http-push.c b/http-push.c index f22f7e43ca..e7bb4cf3d5 100644 --- a/http-push.c +++ b/http-push.c @@ -297,7 +297,7 @@ static void start_mkcol(struct transfer_request *request) request->url = get_remote_object_url(repo->url, hex, 1); - slot = get_active_slot(); + slot = get_active_slot(request->url); slot->callback_func = process_response; slot->callback_data = request; curl_setup_http_get(slot->curl, request->url, DAV_MKCOL); @@ -417,7 +417,7 @@ static void start_put(struct transfer_request *request) strbuf_add(&buf, request->lock->tmpfile_suffix, 41); request->url = strbuf_detach(&buf, NULL); - slot = get_active_slot(); + slot = get_active_slot(request->url); slot->callback_func = process_response; slot->callback_data = request; curl_setup_http(slot->curl, request->url, DAV_PUT, @@ -438,7 +438,7 @@ static void start_move(struct transfer_request *request) struct active_request_slot *slot; struct curl_slist *dav_headers = NULL; - slot = get_active_slot(); + slot = get_active_slot(request->url); slot->callback_func = process_response; slot->callback_data = request; curl_setup_http_get(slot->curl, request->url, DAV_MOVE); @@ -467,7 +467,7 @@ static int refresh_lock(struct remote_lock *lock) dav_headers = get_dav_token_headers(lock, DAV_HEADER_IF | DAV_HEADER_TIMEOUT); - slot = get_active_slot(); + slot = get_active_slot(lock->url); slot->results = &results; curl_setup_http_get(slot->curl, lock->url, DAV_LOCK); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); @@ -882,7 +882,7 @@ static struct remote_lock *lock_remote(const char *path, long timeout) while (ep) { char saved_character = ep[1]; ep[1] = '\0'; - slot = get_active_slot(); + slot = get_active_slot(url); slot->results = &results; curl_setup_http_get(slot->curl, url, DAV_MKCOL); if (start_active_slot(slot)) { @@ -912,7 +912,7 @@ static struct remote_lock *lock_remote(const char *path, long timeout) dav_headers = curl_slist_append(dav_headers, timeout_header); dav_headers = curl_slist_append(dav_headers, "Content-Type: text/xml"); - slot = get_active_slot(); + slot = get_active_slot(url); slot->results = &results; curl_setup_http(slot->curl, url, DAV_LOCK, &out_buffer, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); @@ -980,7 +980,7 @@ static int unlock_remote(struct remote_lock *lock) dav_headers = get_dav_token_headers(lock, DAV_HEADER_LOCK); - slot = get_active_slot(); + slot = get_active_slot(lock->url); slot->results = &results; curl_setup_http_get(slot->curl, lock->url, DAV_UNLOCK); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); @@ -1158,7 +1158,7 @@ static void remote_ls(const char *path, int flags, dav_headers = curl_slist_append(dav_headers, "Depth: 1"); dav_headers = curl_slist_append(dav_headers, "Content-Type: text/xml"); - slot = get_active_slot(); + slot = get_active_slot(url); slot->results = &results; curl_setup_http(slot->curl, url, DAV_PROPFIND, &out_buffer, fwrite_buffer); @@ -1232,7 +1232,7 @@ static int locking_available(void) dav_headers = curl_slist_append(dav_headers, "Depth: 0"); dav_headers = curl_slist_append(dav_headers, "Content-Type: text/xml"); - slot = get_active_slot(); + slot = get_active_slot(repo->url); slot->results = &results; curl_setup_http(slot->curl, repo->url, DAV_PROPFIND, &out_buffer, fwrite_buffer); @@ -1409,7 +1409,7 @@ static int update_remote(unsigned char *sha1, struct remote_lock *lock) strbuf_addf(&out_buffer.buf, "%s\n", sha1_to_hex(sha1)); - slot = get_active_slot(); + slot = get_active_slot(lock->url); slot->results = &results; curl_setup_http(slot->curl, lock->url, DAV_PUT, &out_buffer, fwrite_null); @@ -1535,7 +1535,7 @@ static void update_remote_info_refs(struct remote_lock *lock) if (!aborted) { dav_headers = get_dav_token_headers(lock, DAV_HEADER_IF); - slot = get_active_slot(); + slot = get_active_slot(lock->url); slot->results = &results; curl_setup_http(slot->curl, lock->url, DAV_PUT, &buffer, fwrite_null); @@ -1695,7 +1695,7 @@ static int delete_remote_branch(const char *pattern, int force) return 0; url = xmalloc(strlen(repo->url) + strlen(remote_ref->name) + 1); sprintf(url, "%s%s", repo->url, remote_ref->name); - slot = get_active_slot(); + slot = get_active_slot(url); slot->results = &results; curl_setup_http_get(slot->curl, url, DAV_DELETE); if (start_active_slot(slot)) { diff --git a/http-walker.c b/http-walker.c index 51a906e9e3..5d5ae34fce 100644 --- a/http-walker.c +++ b/http-walker.c @@ -348,7 +348,7 @@ static void fetch_alternates(struct walker *walker, const char *base) * Use a callback to process the result, since another request * may fail and need to have alternates loaded before continuing */ - slot = get_active_slot(); + slot = get_active_slot(url); slot->callback_func = process_alternates_response; alt_req.walker = walker; slot->callback_data = &alt_req; diff --git a/http.c b/http.c index 8ac8eb6c38..d68bbb28cf 100644 --- a/http.c +++ b/http.c @@ -42,6 +42,7 @@ static long curl_low_speed_time = -1; static int curl_ftp_no_epsv; static const char *curl_http_proxy; static const char *curl_cookie_file; +static struct credential cre_url = CREDENTIAL_INIT; static struct credential http_auth = CREDENTIAL_INIT; static int http_proactive_auth; static const char *user_agent; @@ -232,7 +233,7 @@ static int has_cert_password(void) return 1; } -static CURL *get_curl_handle(void) +static CURL *get_curl_handle(const char *url) { CURL *result = curl_easy_init(); @@ -295,6 +296,28 @@ static CURL *get_curl_handle(void) if (curl_ftp_no_epsv) curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0); + if (!curl_http_proxy) { + const char *env_proxy, *no_proxy; + char *env_proxy_var; + int read_http_proxy; + struct strbuf buf = STRBUF_INIT; + credential_from_url(&cre_url, url); + strbuf_addf(&buf, "%s_proxy", cre_url.protocol); + env_proxy_var = strbuf_detach(&buf, NULL); + env_proxy = getenv(env_proxy_var); + if (env_proxy) { + read_http_proxy = 1; + no_proxy = getenv("no_proxy"); + if (!no_proxy) + no_proxy = getenv("NO_PROXY"); + if (no_proxy && (!strcmp("*", no_proxy) || strstr(no_proxy, cre_url.host))) + read_http_proxy = 0; + + if (read_http_proxy) + curl_http_proxy = xstrdup(env_proxy); + } + free(env_proxy_var); + } if (curl_http_proxy) { curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy); curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY); @@ -385,7 +408,7 @@ void http_init(struct remote *remote, const char *url, int proactive_auth) } #ifndef NO_CURL_EASY_DUPHANDLE - curl_default = get_curl_handle(); + curl_default = get_curl_handle(url); #endif } @@ -434,7 +457,7 @@ void http_cleanup(void) ssl_cert_password_required = 0; } -struct active_request_slot *get_active_slot(void) +struct active_request_slot *get_active_slot(const char *url) { struct active_request_slot *slot = active_queue_head; struct active_request_slot *newslot; @@ -472,7 +495,7 @@ struct active_request_slot *get_active_slot(void) if (slot->curl == NULL) { #ifdef NO_CURL_EASY_DUPHANDLE - slot->curl = get_curl_handle(); + slot->curl = get_curl_handle(url); #else slot->curl = curl_easy_duphandle(curl_default); #endif @@ -745,7 +768,7 @@ static int http_request(const char *url, void *result, int target, int options) struct strbuf buf = STRBUF_INIT; int ret; - slot = get_active_slot(); + slot = get_active_slot(url); slot->results = &results; curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1); @@ -1090,7 +1113,7 @@ struct http_pack_request *new_http_pack_request( goto abort; } - preq->slot = get_active_slot(); + preq->slot = get_active_slot(preq->url); curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile); curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite); curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url); @@ -1250,7 +1273,7 @@ struct http_object_request *new_http_object_request(const char *base_url, } } - freq->slot = get_active_slot(); + freq->slot = get_active_slot(freq->url); curl_easy_setopt(freq->slot->curl, CURLOPT_FILE, freq); curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file); diff --git a/http.h b/http.h index 0b61653894..303eafbd2a 100644 --- a/http.h +++ b/http.h @@ -73,7 +73,7 @@ extern curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp); #endif /* Slot lifecycle functions */ -extern struct active_request_slot *get_active_slot(void); +extern struct active_request_slot *get_active_slot(const char *url); extern int start_active_slot(struct active_request_slot *slot); extern void run_active_slot(struct active_request_slot *slot); extern void finish_active_slot(struct active_request_slot *slot); diff --git a/remote-curl.c b/remote-curl.c index d159fe7f34..1b5eaf30b7 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -383,7 +383,7 @@ static int probe_rpc(struct rpc_state *rpc) struct strbuf buf = STRBUF_INIT; int err; - slot = get_active_slot(); + slot = get_active_slot(rpc->service_url); headers = curl_slist_append(headers, rpc->hdr_content_type); headers = curl_slist_append(headers, rpc->hdr_accept); @@ -440,7 +440,7 @@ static int post_rpc(struct rpc_state *rpc) return err; } - slot = get_active_slot(); + slot = get_active_slot(rpc->service_url); curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0); curl_easy_setopt(slot->curl, CURLOPT_POST, 1); From 6e17b8c16aa6fd535ffae04d332e9b7ee8a927d0 Mon Sep 17 00:00:00 2001 From: Nelson Benitez Leon Date: Tue, 13 Mar 2012 15:03:54 +0100 Subject: [PATCH 2/5] http: handle proxy proactive authentication If http_proactive_auth flag is set and there is a username but no password in the proxy url, then interactively ask for the password. This makes possible to not have the password written down in http_proxy env var or in http.proxy config option. Also take care that CURLOPT_PROXY don't include username or password, as we now set them in the new set_proxy_auth() function where we use their specific cURL options. Signed-off-by: Nelson Benitez Leon Signed-off-by: Junio C Hamano --- http.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/http.c b/http.c index d68bbb28cf..7215f4976b 100644 --- a/http.c +++ b/http.c @@ -44,6 +44,7 @@ static const char *curl_http_proxy; static const char *curl_cookie_file; static struct credential cre_url = CREDENTIAL_INIT; static struct credential http_auth = CREDENTIAL_INIT; +static struct credential proxy_auth = CREDENTIAL_INIT; static int http_proactive_auth; static const char *user_agent; @@ -233,6 +234,20 @@ static int has_cert_password(void) return 1; } +static void set_proxy_auth(CURL *result) +{ + if (proxy_auth.username && proxy_auth.password) { +#if LIBCURL_VERSION_NUM >= 0x071901 + curl_easy_setopt(result, CURLOPT_PROXYUSERNAME, proxy_auth.username); + curl_easy_setopt(result, CURLOPT_PROXYPASSWORD, proxy_auth.password); +#else + struct strbuf userpwd = STRBUF_INIT; + strbuf_addf(&userpwd, "%s:%s", proxy_auth.username, proxy_auth.password); + curl_easy_setopt(result, CURLOPT_PROXYUSERPWD, strbuf_detach(&userpwd, NULL)); +#endif + } +} + static CURL *get_curl_handle(const char *url) { CURL *result = curl_easy_init(); @@ -319,8 +334,19 @@ static CURL *get_curl_handle(const char *url) free(env_proxy_var); } if (curl_http_proxy) { - curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy); + struct strbuf proxyhost = STRBUF_INIT; + + if (!proxy_auth.host) /* check to parse only once */ + credential_from_url(&proxy_auth, curl_http_proxy); + + if (http_proactive_auth && proxy_auth.username && !proxy_auth.password) + /* proxy string has username but no password, ask for password */ + credential_fill(&proxy_auth); + + strbuf_addf(&proxyhost, "%s://%s", proxy_auth.protocol, proxy_auth.host); + curl_easy_setopt(result, CURLOPT_PROXY, strbuf_detach(&proxyhost, NULL)); curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + set_proxy_auth(result); } return result; From 4896c4885e160383bfd460a5eaa9c75cb2f70fbc Mon Sep 17 00:00:00 2001 From: Nelson Benitez Leon Date: Tue, 13 Mar 2012 15:04:42 +0100 Subject: [PATCH 3/5] http: handle proxy authentication failure (error 407) Handle http 407 error code by asking for credentials and retrying request in case credentials were not present, or marking credentials as rejected if they were already provided. Signed-off-by: Nelson Benitez Leon Signed-off-by: Junio C Hamano --- http.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/http.c b/http.c index 7215f4976b..85399bf78e 100644 --- a/http.c +++ b/http.c @@ -842,6 +842,15 @@ static int http_request(const char *url, void *result, int target, int options) init_curl_http_auth(slot->curl); ret = HTTP_REAUTH; } + } else if (results.http_code == 407) { /* Proxy authentication failure */ + if (proxy_auth.username && proxy_auth.password) { + credential_reject(&proxy_auth); + ret = HTTP_NOAUTH; + } else { + credential_fill(&proxy_auth); + set_proxy_auth(slot->curl); + ret = HTTP_REAUTH; + } } else { if (!curl_errorstr[0]) strlcpy(curl_errorstr, From 2217c38e5bdc2f8d8356f9a5560d0f2fd4e320c5 Mon Sep 17 00:00:00 2001 From: Nelson Benitez Leon Date: Tue, 13 Mar 2012 15:05:40 +0100 Subject: [PATCH 4/5] http: Avoid limit of retrying request only twice Current code, after receiving HTTP_REAUTH, only retried once, so couldn't do step 3 of the following sequence: 1. We make a request; proxy returns 407, because we didn't give it a password. We ask for the password and return HTTP_REAUTH. 2. We make another request; the proxy passes it to the actual server, who returns 401, because we didn't give an http password. We ask for the password and return HTTP_REAUTH. 3. We make a third request, but this time everybody is happy. Now we retry as long as we keep receiving HTTP_REAUTH, so the previous sequence correctly completes. Patch by Jeff King Signed-off-by: Nelson Benitez Leon Signed-off-by: Junio C Hamano --- http.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/http.c b/http.c index 85399bf78e..e537031108 100644 --- a/http.c +++ b/http.c @@ -875,10 +875,13 @@ static int http_request(const char *url, void *result, int target, int options) static int http_request_reauth(const char *url, void *result, int target, int options) { - int ret = http_request(url, result, target, options); - if (ret != HTTP_REAUTH) - return ret; - return http_request(url, result, target, options); + int ret; + + do { + ret = http_request(url, result, target, options); + } while (ret == HTTP_REAUTH); + + return ret; } int http_get_strbuf(const char *url, struct strbuf *result, int options) From 72d62efe1c6044b25e2e3bdabde3eba2de75db48 Mon Sep 17 00:00:00 2001 From: Nelson Benitez Leon Date: Wed, 14 Mar 2012 12:11:43 +0100 Subject: [PATCH 5/5] http: rename HTTP_REAUTH to HTTP_AUTH_RETRY After adding the proxy authentication support in http, the semantics of HTTP_REAUTH changed more to a retry rather than a re-authentication, so we rename it to HTTP_AUTH_RETRY. Signed-off-by: Nelson Benitez Leon Signed-off-by: Junio C Hamano --- http.c | 6 +++--- http.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/http.c b/http.c index e537031108..1c71edbfdb 100644 --- a/http.c +++ b/http.c @@ -840,7 +840,7 @@ static int http_request(const char *url, void *result, int target, int options) } else { credential_fill(&http_auth); init_curl_http_auth(slot->curl); - ret = HTTP_REAUTH; + ret = HTTP_AUTH_RETRY; } } else if (results.http_code == 407) { /* Proxy authentication failure */ if (proxy_auth.username && proxy_auth.password) { @@ -849,7 +849,7 @@ static int http_request(const char *url, void *result, int target, int options) } else { credential_fill(&proxy_auth); set_proxy_auth(slot->curl); - ret = HTTP_REAUTH; + ret = HTTP_AUTH_RETRY; } } else { if (!curl_errorstr[0]) @@ -879,7 +879,7 @@ static int http_request_reauth(const char *url, void *result, int target, do { ret = http_request(url, result, target, options); - } while (ret == HTTP_REAUTH); + } while (ret == HTTP_AUTH_RETRY); return ret; } diff --git a/http.h b/http.h index 303eafbd2a..6e3ea594be 100644 --- a/http.h +++ b/http.h @@ -123,7 +123,7 @@ extern char *get_remote_object_url(const char *url, const char *hex, #define HTTP_MISSING_TARGET 1 #define HTTP_ERROR 2 #define HTTP_START_FAILED 3 -#define HTTP_REAUTH 4 +#define HTTP_AUTH_RETRY 4 #define HTTP_NOAUTH 5 /*