/* * This file Copyright (C) 2008-2010 Mnemosyne LLC * * This file is licensed by the GPL version 2. Works owned by the * Transmission project are granted a special exemption to clause 2(b) * so that the bulk of its code can remain under the MIT license. * This exemption does not extend to derived works not owned by * the Transmission project. * * $Id$ */ #include #include #include #include #include "transmission.h" #include "net.h" #include "ptrarray.h" #include "session.h" #include "trevent.h" #include "utils.h" #include "version.h" #include "web.h" enum { TR_MEMORY_TRASH = 0xCC, DEFAULT_TIMER_MSEC = 1500, /* arbitrary */ MIN_DNS_CACHE_TIME = 60 * 60 * 24 }; #if 0 #define dbgmsg(...) \ do { \ fprintf( stderr, __VA_ARGS__ ); \ fprintf( stderr, "\n" ); \ } while( 0 ) #else #define dbgmsg( ... ) \ do { \ if( tr_deepLoggingIsActive( ) ) \ tr_deepLog( __FILE__, __LINE__, "web", __VA_ARGS__ ); \ } while( 0 ) #endif /*** **** ***/ struct tr_web { tr_bool closing; tr_bool haveAddr; int taskCount; long timer_msec; CURLM * multi; tr_session * session; tr_address addr; tr_ptrArray dns_cache; struct event timer_event; }; struct dns_cache_item; static void dns_cache_item_free( struct dns_cache_item * ); static void web_free( tr_web * g ) { evdns_shutdown( TRUE ); curl_multi_cleanup( g->multi ); evtimer_del( &g->timer_event ); tr_ptrArrayDestruct( &g->dns_cache, (PtrArrayForeachFunc)dns_cache_item_free ); memset( g, TR_MEMORY_TRASH, sizeof( struct tr_web ) ); tr_free( g ); } /*** **** ***/ struct tr_web_task { int port; unsigned long tag; struct curl_slist * slist; struct evbuffer * response; char * url; char * host; const char * resolved_host; char * range; tr_session * session; tr_web_done_func * done_func; void * done_func_user_data; struct event timer_event; CURL * easy; CURLM * multi; tr_bool timer_event_isSet; }; static void task_free( struct tr_web_task * task ) { if( task->slist != NULL ) curl_slist_free_all( task->slist ); if( task->timer_event_isSet ) evtimer_del( &task->timer_event ); evbuffer_free( task->response ); tr_free( task->host ); tr_free( task->range ); tr_free( task->url ); memset( task, TR_MEMORY_TRASH, sizeof( struct tr_web_task ) ); tr_free( task ); } /*** **** ***/ struct dns_cache_item { char * host; char * resolved_host; time_t expiration; tr_bool success; }; static void dns_cache_item_free( struct dns_cache_item * item ) { if( item != NULL ) { tr_free( item->host ); tr_free( item->resolved_host ); memset( item, TR_MEMORY_TRASH, sizeof( struct dns_cache_item ) ); tr_free( item ); } } static int dns_cache_compare( const void * va, const void * vb ) { const struct dns_cache_item * a = va; const struct dns_cache_item * b = vb; return strcmp( a->host, b->host ); } typedef enum { TR_DNS_OK, TR_DNS_FAIL, TR_DNS_UNTESTED } tr_dns_result; static void dns_cache_clear_entry( struct tr_ptrArray * cache, const char * host ) { struct dns_cache_item key; key.host = (char*) host; dns_cache_item_free( tr_ptrArrayRemoveSorted( cache, &key, dns_cache_compare ) ); } static tr_dns_result dns_cache_lookup( struct tr_web_task * task, const char * host, const char ** resolved ) { tr_dns_result result = TR_DNS_UNTESTED; if( task->session->web != NULL ) { struct dns_cache_item key; struct dns_cache_item * item; tr_ptrArray * cache = &task->session->web->dns_cache; key.host = (char*) host; item = tr_ptrArrayFindSorted( cache, &key, dns_cache_compare ); /* has the ttl expired? */ if( ( item != NULL ) && ( item->expiration <= tr_time( ) ) ) { dns_cache_clear_entry( cache, host ); item = NULL; } if( item != NULL ) { result = item->success ? TR_DNS_OK : TR_DNS_FAIL; if( result == TR_DNS_OK ) { *resolved = item->resolved_host; dbgmsg( "found cached dns entry for \"%s\": %s", host, *resolved ); } } } return result; } static void dns_cache_set_fail( struct tr_web_task * task, const char * host ) { if( task->session->web != NULL ) { struct dns_cache_item * item; tr_ptrArray * cache = &task->session->web->dns_cache; dns_cache_clear_entry( cache, host ); item = tr_new( struct dns_cache_item, 1 ); item->host = tr_strdup( host ); item->resolved_host = NULL; item->expiration = tr_time( ) + MIN_DNS_CACHE_TIME; item->success = FALSE; tr_ptrArrayInsertSorted( cache, item, dns_cache_compare ); } } static const char* dns_cache_set_name( struct tr_web_task * task, const char * host, const char * resolved, int ttl ) { char * ret = NULL; ttl = MAX( MIN_DNS_CACHE_TIME, ttl ); if( task->session->web != NULL ) { struct dns_cache_item * item; tr_ptrArray * cache = &task->session->web->dns_cache; dns_cache_clear_entry( cache, host ); item = tr_new( struct dns_cache_item, 1 ); item->host = tr_strdup( host ); item->resolved_host = tr_strdup( resolved ); item->expiration = tr_time( ) + ttl; item->success = TRUE; tr_ptrArrayInsertSorted( cache, item, dns_cache_compare ); ret = item->resolved_host; dbgmsg( "adding dns cache entry for \"%s\": %s", host, resolved ); } return ret; } /*** **** ***/ static size_t writeFunc( void * ptr, size_t size, size_t nmemb, void * vtask ) { const size_t byteCount = size * nmemb; struct tr_web_task * task = vtask; evbuffer_add( task->response, ptr, byteCount ); dbgmsg( "wrote %zu bytes to task %p's buffer", byteCount, task ); return byteCount; } static int sockoptfunction( void * vtask, curl_socket_t fd, curlsocktype purpose UNUSED ) { struct tr_web_task * task = vtask; const tr_bool isScrape = strstr( task->url, "scrape" ) != NULL; const tr_bool isAnnounce = strstr( task->url, "announce" ) != NULL; /* announce and scrape requests have tiny payloads. */ if( isScrape || isAnnounce ) { const int sndbuf = 1024; const int rcvbuf = isScrape ? 2048 : 3072; setsockopt( fd, SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf) ); setsockopt( fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf) ); } /* return nonzero if this function encountered an error */ return 0; } static int getCurlProxyType( tr_proxy_type t ) { if( t == TR_PROXY_SOCKS4 ) return CURLPROXY_SOCKS4; if( t == TR_PROXY_SOCKS5 ) return CURLPROXY_SOCKS5; return CURLPROXY_HTTP; } static int getTimeoutFromURL( const char * url ) { if( strstr( url, "scrape" ) != NULL ) return 20; if( strstr( url, "announce" ) != NULL ) return 45; return 240; } static void task_timeout_cb( int fd UNUSED, short what UNUSED, void * task ); static void task_finish( struct tr_web_task * task, long response_code ); static void addTask( void * vtask ) { struct tr_web_task * task = vtask; const tr_session * session = task->session; if( ( session == NULL ) || ( session->web == NULL ) ) return; if( !task->resolved_host ) { dbgmsg( "couldn't resolve host for \"%s\"... task failed", task->url ); task_finish( task, 0 ); } else { CURL * e = curl_easy_init( ); struct tr_web * web = session->web; const int timeout = getTimeoutFromURL( task->url ); const long verbose = getenv( "TR_CURL_VERBOSE" ) != NULL; const char * user_agent = TR_NAME "/" LONG_VERSION_STRING; char * url = NULL; /* insert the resolved host into the URL s.t. curl's DNS won't block * even if -- like on most OSes -- it wasn't built with C-Ares :( * "http://www.craptrackular.org/announce?key=val&key2=..." becomes * "http://127.0.0.1/announce?key=val&key2=..." */ { char * host; struct evbuffer * buf = evbuffer_new( ); char * pch = strstr( task->url, task->host ); char * tail = pch + strlen( task->host ); evbuffer_add( buf, task->url, pch - task->url ); evbuffer_add_printf( buf, "%s", task->resolved_host ); evbuffer_add_printf( buf, "%s", tail ); url = tr_strndup( EVBUFFER_DATA( buf ), EVBUFFER_LENGTH( buf ) ); dbgmsg( "old url: \"%s\" -- new url: \"%s\"", task->url, url ); evbuffer_free( buf ); /* Manually add a Host: argument that refers to the true URL */ if( ( ( task->port <= 0 ) ) || ( ( task->port == 80 ) && !strncmp( task->url, "http://", 7 ) ) || ( ( task->port == 443 ) && !strncmp( task->url, "https://", 8 ) ) ) host = tr_strdup_printf( "Host: %s", task->host ); else host = tr_strdup_printf( "Host: %s:%d", task->host, task->port ); task->slist = curl_slist_append( NULL, host ); curl_easy_setopt( e, CURLOPT_HTTPHEADER, task->slist ); tr_free( host ); } dbgmsg( "adding task #%lu [%s]", task->tag, url ? url : task->url ); if( !task->range && session->isProxyEnabled ) { curl_easy_setopt( e, CURLOPT_PROXY, session->proxy ); curl_easy_setopt( e, CURLOPT_PROXYAUTH, CURLAUTH_ANY ); curl_easy_setopt( e, CURLOPT_PROXYPORT, session->proxyPort ); curl_easy_setopt( e, CURLOPT_PROXYTYPE, getCurlProxyType( session->proxyType ) ); } if( !task->range && session->isProxyAuthEnabled ) { char * str = tr_strdup_printf( "%s:%s", session->proxyUsername, session->proxyPassword ); curl_easy_setopt( e, CURLOPT_PROXYUSERPWD, str ); tr_free( str ); } task->easy = e; task->multi = web->multi; /* use our own timeout instead of CURLOPT_TIMEOUT because the latter * doesn't play nicely with curl_multi. See curl bug #2501457 */ task->timer_event_isSet = TRUE; evtimer_set( &task->timer_event, task_timeout_cb, task ); tr_timerAdd( &task->timer_event, timeout, 0 ); curl_easy_setopt( e, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ); curl_easy_setopt( e, CURLOPT_SOCKOPTFUNCTION, sockoptfunction ); curl_easy_setopt( e, CURLOPT_SOCKOPTDATA, task ); curl_easy_setopt( e, CURLOPT_WRITEDATA, task ); curl_easy_setopt( e, CURLOPT_WRITEFUNCTION, writeFunc ); curl_easy_setopt( e, CURLOPT_DNS_CACHE_TIMEOUT, MIN_DNS_CACHE_TIME ); curl_easy_setopt( e, CURLOPT_FOLLOWLOCATION, 1L ); curl_easy_setopt( e, CURLOPT_AUTOREFERER, 1L ); curl_easy_setopt( e, CURLOPT_FORBID_REUSE, 1L ); curl_easy_setopt( e, CURLOPT_MAXREDIRS, -1L ); curl_easy_setopt( e, CURLOPT_NOSIGNAL, 1L ); curl_easy_setopt( e, CURLOPT_PRIVATE, task ); curl_easy_setopt( e, CURLOPT_SSL_VERIFYHOST, 0L ); curl_easy_setopt( e, CURLOPT_SSL_VERIFYPEER, 0L ); curl_easy_setopt( e, CURLOPT_URL, url ? url : task->url ); curl_easy_setopt( e, CURLOPT_USERAGENT, user_agent ); curl_easy_setopt( e, CURLOPT_VERBOSE, verbose ); if( web->haveAddr ) curl_easy_setopt( e, CURLOPT_INTERFACE, tr_ntop_non_ts( &web->addr ) ); if( task->range ) curl_easy_setopt( e, CURLOPT_RANGE, task->range ); if( curl_multi_add_handle( web->multi, e ) == CURLM_OK ) ++web->taskCount; tr_free( url ); } } static void dns_ipv6_done_cb( int err, char type, int count, int ttl, void * addresses, void * vtask ) { struct tr_web_task * task = vtask; if( !err && task->host && ( count>0 ) && ( ttl>=0 ) && ( type==DNS_IPv6_AAAA ) ) { int i; char buf[INET6_ADDRSTRLEN+1]; struct in6_addr *in6_addrs = addresses; for( i=0; i 1? */ task->resolved_host = dns_cache_set_name( task, task->host, b, ttl ); break; } } } if( task->resolved_host == NULL ) dns_cache_set_fail( task, task->host ); addTask( task ); } static void dns_ipv4_done_cb( int err, char type, int count, int ttl, void * addresses, void * vtask ) { struct tr_web_task * task = vtask; if( !err && task->host && ( count>0 ) && ( ttl>=0 ) && ( type==DNS_IPv4_A ) ) { struct in_addr * in_addrs = addresses; const char * resolved = inet_ntoa( in_addrs[0] ); task->resolved_host = dns_cache_set_name( task, task->host, resolved, ttl ); /* FIXME: if count > 1, is there a way to decide which is best to use? */ } if( ( task->resolved_host != NULL ) || ( task->host == NULL ) || evdns_resolve_ipv6( task->host, 0, dns_ipv6_done_cb, task ) ) dns_ipv6_done_cb( DNS_ERR_UNKNOWN, DNS_IPv6_AAAA, 0, 0, NULL, task ); } static void doDNS( void * vtask ) { tr_address addr; int port = -1; char * host = NULL; struct tr_web_task * task = vtask; tr_dns_result lookup_result = TR_DNS_UNTESTED; assert( task->resolved_host == NULL ); if( !tr_httpParseURL( task->url, -1, &host, &port, NULL ) ) { task->port = port; task->host = host; /* If 'host' is an IPv4 or IPv6 address in text form, use it as-is. * Otherwise, see if its resolved name is in our DNS cache */ if( tr_pton( task->host, &addr ) != NULL ) { task->resolved_host = task->host; lookup_result = TR_DNS_OK; } else { lookup_result = dns_cache_lookup( task, host, &task->resolved_host ); } } if( lookup_result != TR_DNS_UNTESTED ) { addTask( task ); } else if( !host || evdns_resolve_ipv4( host, 0, dns_ipv4_done_cb, task ) ) { dns_ipv4_done_cb( DNS_ERR_UNKNOWN, DNS_IPv4_A, 0, 0, NULL, task ); } } /*** **** ***/ static void task_finish( struct tr_web_task * task, long response_code ) { dbgmsg( "finished web task %lu; got %ld", task->tag, response_code ); if( task->done_func != NULL ) task->done_func( task->session, response_code, EVBUFFER_DATA( task->response ), EVBUFFER_LENGTH( task->response ), task->done_func_user_data ); task_free( task ); } static void remove_task( struct tr_web_task * task ) { long code; curl_easy_getinfo( task->easy, CURLINFO_RESPONSE_CODE, &code ); curl_multi_remove_handle( task->multi, task->easy ); curl_easy_cleanup( task->easy ); task_finish( task, code ); } static void task_timeout_cb( int fd UNUSED, short what UNUSED, void * task ) { remove_task( task ); } static void remove_finished_tasks( tr_web * g ) { CURLMsg * msg; int msgs_left; while(( msg = curl_multi_info_read( g->multi, &msgs_left ))) { if(( msg->msg == CURLMSG_DONE ) && ( msg->easy_handle != NULL )) { struct tr_web_task * task; CURL * e = msg->easy_handle; curl_easy_getinfo( e, CURLINFO_PRIVATE, (void*)&task ); assert( e == task->easy ); remove_task( task ); } } } static void restart_timer( tr_web * g ) { dbgmsg( "adding a timeout for %.1f seconds from now", g->timer_msec/1000.0 ); evtimer_del( &g->timer_event ); tr_timerAddMsec( &g->timer_event, g->timer_msec ); } static void tr_multi_perform( tr_web * g, int fd, int curl_what ) { CURLMcode m; dbgmsg( "check_run_count: %d taskCount", g->taskCount ); /* invoke libcurl's processing */ do m = curl_multi_socket_action( g->multi, fd, curl_what, &g->taskCount ); while( m == CURLM_CALL_MULTI_SOCKET ); remove_finished_tasks( g ); if( g->closing && !g->taskCount ) web_free( g ); else restart_timer( g ); } /* libevent says that sock is ready to be processed, so wake up libcurl */ static void event_cb( int fd, short ev_what, void * g ) { int curl_what = 0; if( ev_what & EV_READ ) curl_what |= CURL_POLL_IN; if( ev_what & EV_WRITE ) curl_what |= CURL_POLL_OUT; tr_multi_perform( g, fd, curl_what ); } /* CURLMOPT_SOCKETFUNCTION */ static int sock_cb( CURL * e UNUSED, curl_socket_t fd, int curl_what, void * vweb, void * vevent ) { /*static int num_events = 0;*/ struct tr_web * web = vweb; struct event * io_event = vevent; dbgmsg( "sock_cb: curl_what %d, fd %d, io_event %p", curl_what, (int)fd, io_event ); if( io_event != NULL ) event_del( io_event ); if( curl_what & ( CURL_POLL_IN | CURL_POLL_OUT ) ) { const short ev_what = EV_PERSIST | (( curl_what & CURL_POLL_IN ) ? EV_READ : 0 ) | (( curl_what & CURL_POLL_OUT ) ? EV_WRITE : 0 ); if( io_event == NULL ) { io_event = tr_new0( struct event, 1 ); curl_multi_assign( web->multi, fd, io_event ); /*fprintf( stderr, "+1 io_events to %d\n", ++num_events );*/ } dbgmsg( "enabling (libevent %hd, libcurl %d) on io_event %p, fd %d", ev_what, curl_what, io_event, fd ); event_set( io_event, fd, ev_what, event_cb, web ); assert( io_event->ev_base != NULL ); event_add( io_event, NULL ); } if( ( io_event != NULL ) && ( curl_what & CURL_POLL_REMOVE ) ) { memset( io_event, TR_MEMORY_TRASH, sizeof( struct event ) ); tr_free( io_event ); /*fprintf( stderr, "-1 io_events to %d\n", --num_events );*/ } return 0; /* libcurl documentation: "The callback MUST return 0." */ } /* libevent says that timer_msec have passed, so wake up libcurl */ static void libevent_timer_cb( int fd UNUSED, short what UNUSED, void * g ) { dbgmsg( "libevent timer is done" ); tr_multi_perform( g, CURL_SOCKET_TIMEOUT, 0 ); } /* libcurl documentation: "If 0, it means you should proceed immediately * without waiting for anything. If it returns -1, there's no timeout at all * set ... (but) you must not wait too long (more than a few seconds perhaps) * before you call curl_multi_perform() again." */ static void multi_timer_cb( CURLM * multi UNUSED, long timer_msec, void * vg ) { tr_web * g = vg; g->timer_msec = timer_msec > 0 ? timer_msec : DEFAULT_TIMER_MSEC; if( timer_msec < 1 ) tr_multi_perform( g, CURL_SOCKET_TIMEOUT, 0 ); else restart_timer( g ); } /**** ***** ****/ void tr_webRun( tr_session * session, const char * url, const char * range, tr_web_done_func done_func, void * done_func_user_data ) { if( session->web != NULL ) { static unsigned long tag = 0; struct tr_web_task * task = tr_new0( struct tr_web_task, 1 ); task->session = session; task->url = tr_strdup( url ); task->range = tr_strdup( range ); task->done_func = done_func; task->done_func_user_data = done_func_user_data; task->tag = ++tag; task->response = evbuffer_new( ); tr_runInEventThread( session, doDNS, task ); } } void tr_webSetInterface( tr_web * web, const tr_address * addr ) { if(( web->haveAddr = ( addr != NULL ))) web->addr = *addr; } tr_web* tr_webInit( tr_session * session ) { tr_web * web; /* try to enable ssl for https support; but if that fails, * try a plain vanilla init */ if( curl_global_init( CURL_GLOBAL_SSL ) ) curl_global_init( 0 ); web = tr_new0( struct tr_web, 1 ); web->dns_cache = TR_PTR_ARRAY_INIT; web->session = session; web->timer_msec = DEFAULT_TIMER_MSEC; /* overwritten by multi_timer_cb() */ evtimer_set( &web->timer_event, libevent_timer_cb, web ); web->multi = curl_multi_init( ); evdns_init( ); curl_multi_setopt( web->multi, CURLMOPT_SOCKETDATA, web ); curl_multi_setopt( web->multi, CURLMOPT_SOCKETFUNCTION, sock_cb ); curl_multi_setopt( web->multi, CURLMOPT_TIMERDATA, web ); curl_multi_setopt( web->multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb ); return web; } void tr_webClose( tr_web ** web_in ) { tr_web * web = *web_in; *web_in = NULL; if( web->taskCount < 1 ) web_free( web ); else web->closing = 1; } /***** ****** ****** *****/ const char * tr_webGetResponseStr( long code ) { switch( code ) { case 0: return "No Response"; case 101: return "Switching Protocols"; case 200: return "OK"; case 201: return "Created"; case 202: return "Accepted"; case 203: return "Non-Authoritative Information"; case 204: return "No Content"; case 205: return "Reset Content"; case 206: return "Partial Content"; case 300: return "Multiple Choices"; case 301: return "Moved Permanently"; case 302: return "Found"; case 303: return "See Other"; case 304: return "Not Modified"; case 305: return "Use Proxy"; case 306: return "(Unused)"; case 307: return "Temporary Redirect"; case 400: return "Bad Request"; case 401: return "Unauthorized"; case 402: return "Payment Required"; case 403: return "Forbidden"; case 404: return "Not Found"; case 405: return "Method Not Allowed"; case 406: return "Not Acceptable"; case 407: return "Proxy Authentication Required"; case 408: return "Request Timeout"; case 409: return "Conflict"; case 410: return "Gone"; case 411: return "Length Required"; case 412: return "Precondition Failed"; case 413: return "Request Entity Too Large"; case 414: return "Request-URI Too Long"; case 415: return "Unsupported Media Type"; case 416: return "Requested Range Not Satisfiable"; case 417: return "Expectation Failed"; case 500: return "Internal Server Error"; case 501: return "Not Implemented"; case 502: return "Bad Gateway"; case 503: return "Service Unavailable"; case 504: return "Gateway Timeout"; case 505: return "HTTP Version Not Supported"; default: return "Unknown Error"; } } void tr_http_escape( struct evbuffer * out, const char * str, int len, tr_bool escape_slashes ) { int i; if( ( len < 0 ) && ( str != NULL ) ) len = strlen( str ); for( i = 0; i < len; i++ ) { if( str[i] == ',' || str[i] == '-' || str[i] == '.' || ( '0' <= str[i] && str[i] <= '9' ) || ( 'A' <= str[i] && str[i] <= 'Z' ) || ( 'a' <= str[i] && str[i] <= 'z' ) || ( str[i] == '/' && !escape_slashes ) ) evbuffer_add( out, &str[i], 1 ); else evbuffer_add_printf( out, "%%%02X", (unsigned)(str[i]&0xFF) ); } } char * tr_http_unescape( const char * str, int len ) { char * tmp = curl_unescape( str, len ); char * ret = tr_strdup( tmp ); curl_free( tmp ); return ret; }