1#include "http_curl.h"
2#if !defined(CONF_PLATFORM_EMSCRIPTEN)
3
4#include <base/dbg.h>
5#include <base/log.h>
6#include <base/str.h>
7#include <base/thread.h>
8
9#include <engine/shared/config.h>
10#include <engine/storage.h>
11
12#include <cstring>
13#include <limits>
14
15#if !defined(CONF_FAMILY_WINDOWS)
16#include <csignal>
17#endif
18
19#include <curl/curl.h>
20
21static int CurlDebug(CURL *pHandle, curl_infotype Type, char *pData, size_t DataSize, void *pUser)
22{
23 char TypeChar;
24 switch(Type)
25 {
26 case CURLINFO_TEXT:
27 TypeChar = '*';
28 break;
29 case CURLINFO_HEADER_OUT:
30 TypeChar = '<';
31 break;
32 case CURLINFO_HEADER_IN:
33 TypeChar = '>';
34 break;
35 default:
36 return 0;
37 }
38 while(const char *pLineEnd = (const char *)memchr(s: pData, c: '\n', n: DataSize))
39 {
40 int LineLength = pLineEnd - pData;
41 log_debug("curl", "%c %.*s", TypeChar, LineLength, pData);
42 pData += LineLength + 1;
43 DataSize -= LineLength + 1;
44 }
45 return 0;
46}
47
48CHttpRequestCurl::CHttpRequestCurl(const char *pUrl) :
49 IHttpRequest(pUrl)
50{
51}
52
53CHttpRequestCurl::~CHttpRequestCurl()
54{
55 dbg_assert(m_File == nullptr, "HTTP request file was not closed");
56 curl_slist_free_all(list: m_pRequestHeaders);
57}
58
59void CHttpRequestCurl::Header(const char *pNameColonValue)
60{
61 m_pRequestHeaders = curl_slist_append(list: m_pRequestHeaders, data: pNameColonValue);
62}
63
64bool CHttpRequestCurl::ConfigureHandle(CURL *pHandle)
65{
66 if(!BeforeInit())
67 {
68 return false;
69 }
70
71 if(g_Config.m_DbgHttp)
72 {
73 curl_easy_setopt(pHandle, CURLOPT_VERBOSE, 1L);
74 curl_easy_setopt(pHandle, CURLOPT_DEBUGFUNCTION, CurlDebug);
75 }
76 long Protocols = CURLPROTO_HTTPS;
77 if(g_Config.m_HttpAllowInsecure)
78 {
79 Protocols |= CURLPROTO_HTTP;
80 }
81
82 curl_easy_setopt(pHandle, CURLOPT_ERRORBUFFER, m_aErr);
83
84 curl_easy_setopt(pHandle, CURLOPT_CONNECTTIMEOUT_MS, m_Timeout.m_ConnectTimeoutMs);
85 curl_easy_setopt(pHandle, CURLOPT_TIMEOUT_MS, m_Timeout.m_TimeoutMs);
86 curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_LIMIT, m_Timeout.m_LowSpeedLimit);
87 curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_TIME, m_Timeout.m_LowSpeedTime);
88 if(m_MaxResponseSize >= 0)
89 {
90 curl_easy_setopt(pHandle, CURLOPT_MAXFILESIZE_LARGE, (curl_off_t)m_MaxResponseSize);
91 }
92 if(m_IfModifiedSince >= 0)
93 {
94 curl_easy_setopt(pHandle, CURLOPT_TIMEVALUE_LARGE, (curl_off_t)m_IfModifiedSince);
95 curl_easy_setopt(pHandle, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
96 }
97
98 // ‘CURLOPT_PROTOCOLS’ is deprecated: since 7.85.0. Use CURLOPT_PROTOCOLS_STR
99 // Wait until all platforms have 7.85.0
100#ifdef __GNUC__
101#pragma GCC diagnostic push
102#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
103#endif
104 curl_easy_setopt(pHandle, CURLOPT_PROTOCOLS, Protocols);
105#ifdef __GNUC__
106#pragma GCC diagnostic pop
107#endif
108 curl_easy_setopt(pHandle, CURLOPT_FOLLOWLOCATION, 1L);
109 curl_easy_setopt(pHandle, CURLOPT_MAXREDIRS, 4L);
110 if(m_FailOnErrorStatus)
111 {
112 curl_easy_setopt(pHandle, CURLOPT_FAILONERROR, 1L);
113 }
114 curl_easy_setopt(pHandle, CURLOPT_URL, m_aUrl);
115 curl_easy_setopt(pHandle, CURLOPT_NOSIGNAL, 1L);
116 curl_easy_setopt(pHandle, CURLOPT_USERAGENT, USER_AGENT_STRING);
117 curl_easy_setopt(pHandle, CURLOPT_ACCEPT_ENCODING, ""); // Use any compression algorithm supported by libcurl.
118
119 curl_easy_setopt(pHandle, CURLOPT_HEADERDATA, this);
120 curl_easy_setopt(pHandle, CURLOPT_HEADERFUNCTION, HeaderCallback);
121 curl_easy_setopt(pHandle, CURLOPT_WRITEDATA, this);
122 curl_easy_setopt(pHandle, CURLOPT_WRITEFUNCTION, WriteCallback);
123 curl_easy_setopt(pHandle, CURLOPT_NOPROGRESS, 0L);
124 curl_easy_setopt(pHandle, CURLOPT_PROGRESSDATA, this);
125 // ‘CURLOPT_PROGRESSFUNCTION’ is deprecated: since 7.32.0. Use CURLOPT_XFERINFOFUNCTION
126 // See problems with curl_off_t type in header file in https://github.com/ddnet/ddnet/pull/6185/
127#ifdef __GNUC__
128#pragma GCC diagnostic push
129#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
130#endif
131 curl_easy_setopt(pHandle, CURLOPT_PROGRESSFUNCTION, ProgressCallback);
132#ifdef __GNUC__
133#pragma GCC diagnostic pop
134#endif
135 curl_easy_setopt(pHandle, CURLOPT_IPRESOLVE, m_IpResolve == IPRESOLVE::V4 ? CURL_IPRESOLVE_V4 : (m_IpResolve == IPRESOLVE::V6 ? CURL_IPRESOLVE_V6 : CURL_IPRESOLVE_WHATEVER));
136 if(g_Config.m_Bindaddr[0] != '\0')
137 {
138 curl_easy_setopt(pHandle, CURLOPT_INTERFACE, g_Config.m_Bindaddr);
139 }
140
141 if(curl_version_info(CURLVERSION_NOW)->version_num < 0x074400)
142 {
143 // Causes crashes, see https://github.com/ddnet/ddnet/issues/4342.
144 // No longer a problem in curl 7.68 and above, and 0x44 = 68.
145 curl_easy_setopt(pHandle, CURLOPT_FORBID_REUSE, 1L);
146 }
147
148#ifdef CONF_PLATFORM_ANDROID
149 curl_easy_setopt(pHandle, CURLOPT_CAPATH, "/system/etc/security/cacerts");
150#endif
151
152 switch(m_Type)
153 {
154 case REQUEST::GET:
155 break;
156 case REQUEST::HEAD:
157 curl_easy_setopt(pHandle, CURLOPT_NOBODY, 1L);
158 break;
159 case REQUEST::POST:
160 case REQUEST::POST_JSON:
161 if(m_Type == REQUEST::POST_JSON)
162 {
163 Header(pNameColonValue: "Content-Type: application/json");
164 }
165 else
166 {
167 Header(pNameColonValue: "Content-Type:");
168 }
169 curl_easy_setopt(pHandle, CURLOPT_POSTFIELDS, m_pBody);
170 curl_easy_setopt(pHandle, CURLOPT_POSTFIELDSIZE, m_BodyLength);
171 break;
172 }
173
174 curl_easy_setopt(pHandle, CURLOPT_HTTPHEADER, m_pRequestHeaders);
175
176 return true;
177}
178
179size_t CHttpRequestCurl::OnHeader(char *pHeader, size_t HeaderSize)
180{
181 // `pHeader` is NOT null-terminated.
182 // `pHeader` has a trailing newline.
183
184 if(HeaderSize <= 1)
185 {
186 m_ResponseHeadersEnded = true;
187 return HeaderSize;
188 }
189 if(m_ResponseHeadersEnded)
190 {
191 // redirect, clear old headers
192 m_ResponseHeadersEnded = false;
193 m_ResultDate = {};
194 m_ResultLastModified = {};
195 }
196
197 static const char DATE[] = "Date: ";
198 static const char LAST_MODIFIED[] = "Last-Modified: ";
199
200 // Trailing newline and null termination evens out.
201 if(HeaderSize - 1 >= sizeof(DATE) - 1 && str_startswith_nocase(str: pHeader, prefix: DATE))
202 {
203 char aValue[128];
204 str_truncate(dst: aValue, dst_size: sizeof(aValue), src: pHeader + (sizeof(DATE) - 1), truncation_len: HeaderSize - (sizeof(DATE) - 1) - 1);
205 int64_t Value = curl_getdate(p: aValue, unused: nullptr);
206 if(Value != -1)
207 {
208 m_ResultDate = Value;
209 }
210 }
211 if(HeaderSize - 1 >= sizeof(LAST_MODIFIED) - 1 && str_startswith_nocase(str: pHeader, prefix: LAST_MODIFIED))
212 {
213 char aValue[128];
214 str_truncate(dst: aValue, dst_size: sizeof(aValue), src: pHeader + (sizeof(LAST_MODIFIED) - 1), truncation_len: HeaderSize - (sizeof(LAST_MODIFIED) - 1) - 1);
215 int64_t Value = curl_getdate(p: aValue, unused: nullptr);
216 if(Value != -1)
217 {
218 m_ResultLastModified = Value;
219 }
220 }
221
222 return HeaderSize;
223}
224
225void CHttpRequestCurl::OnCompletionInternal(CURL *pHandle, CURLcode Code)
226{
227 if(pHandle)
228 {
229 long StatusCode;
230 curl_easy_getinfo(pHandle, CURLINFO_RESPONSE_CODE, &StatusCode);
231 m_StatusCode = StatusCode;
232 }
233
234 EHttpState State;
235 if(Code != CURLE_OK)
236 {
237 if(g_Config.m_DbgHttp || m_LogProgress >= HTTPLOG::FAILURE)
238 {
239 log_error("http", "%s failed. libcurl error (%u): %s", m_aUrl, Code, m_aErr[0] != '\0' ? m_aErr : curl_easy_strerror(Code));
240 }
241 State = (Code == CURLE_ABORTED_BY_CALLBACK) ? EHttpState::ABORTED : EHttpState::ERROR;
242 }
243 else
244 {
245 if(g_Config.m_DbgHttp || m_LogProgress >= HTTPLOG::ALL)
246 {
247 log_info("http", "task done: %s", m_aUrl);
248 }
249 State = EHttpState::DONE;
250 }
251
252 IHttpRequest::OnCompletionInternal(State);
253}
254
255size_t CHttpRequestCurl::HeaderCallback(char *pData, size_t Size, size_t Number, void *pUser)
256{
257 dbg_assert(Size == 1, "invalid size parameter passed to header callback");
258 return ((CHttpRequestCurl *)pUser)->OnHeader(pHeader: pData, HeaderSize: Number);
259}
260
261size_t CHttpRequestCurl::WriteCallback(char *pData, size_t Size, size_t Number, void *pUser)
262{
263 return ((CHttpRequestCurl *)pUser)->OnData(pData, DataSize: Size * Number);
264}
265
266int CHttpRequestCurl::ProgressCallback(void *pUser, double DlTotal, double DlCurr, double UlTotal, double UlCurr)
267{
268 CHttpRequestCurl *pTask = (CHttpRequestCurl *)pUser;
269 pTask->m_Current.store(t: DlCurr, m: std::memory_order_relaxed);
270 pTask->m_Size.store(t: DlTotal, m: std::memory_order_relaxed);
271 pTask->m_Progress.store(i: DlTotal == 0.0 ? 0 : (100 * DlCurr) / DlTotal, m: std::memory_order_relaxed);
272 if(pTask->m_pProgressCallback != nullptr)
273 {
274 pTask->m_pProgressCallback->OnProgress();
275 }
276 return pTask->m_Abort ? -1 : 0;
277}
278
279std::unique_ptr<IHttpRequest> CreateHttpRequest(const char *pUrl)
280{
281 return std::make_unique<CHttpRequestCurl>(args&: pUrl);
282}
283
284void EscapeUrl(char *pBuf, size_t Size, const char *pStr)
285{
286 char *pEsc = curl_easy_escape(curl: nullptr, string: pStr, length: 0);
287 str_copy(dst: pBuf, src: pEsc, dst_size: Size);
288 curl_free(p: pEsc);
289}
290
291bool CHttpCurl::Init(std::chrono::milliseconds ShutdownDelay)
292{
293 m_ShutdownDelay = ShutdownDelay;
294
295#if !defined(CONF_FAMILY_WINDOWS)
296 // As a multithreaded application we have to tell curl to not install signal
297 // handlers and instead ignore SIGPIPE from OpenSSL ourselves.
298 signal(SIGPIPE, SIG_IGN);
299#endif
300 m_pThread = thread_init(threadfunc: CHttpCurl::ThreadMain, user: this, name: "http");
301
302 std::unique_lock Lock(m_Lock);
303 m_ConditionVariableInit.wait(lock&: Lock, p: [this]() { return m_State != CHttpCurl::UNINITIALIZED; });
304 if(m_State != CHttpCurl::RUNNING)
305 {
306 return false;
307 }
308
309 return true;
310}
311
312void CHttpCurl::Shutdown()
313{
314 std::unique_lock Lock(m_Lock);
315 if(m_Shutdown || m_State != CHttpCurl::RUNNING)
316 return;
317
318 m_Shutdown = true;
319 curl_multi_wakeup(m: m_pMultiH);
320}
321
322CHttpCurl::~CHttpCurl()
323{
324 if(!m_pThread)
325 return;
326
327 Shutdown();
328 thread_wait(thread: m_pThread);
329}
330
331void CHttpCurl::Run(std::shared_ptr<IHttpRequest> pRequest)
332{
333 std::shared_ptr<CHttpRequestCurl> pRequestImpl = std::static_pointer_cast<CHttpRequestCurl>(r: pRequest);
334 std::unique_lock Lock(m_Lock);
335 if(m_Shutdown || m_State == CHttpCurl::ERROR)
336 {
337 str_copy(dst&: pRequestImpl->m_aErr, src: "Shutting down");
338 pRequestImpl->OnCompletionInternal(pHandle: nullptr, Code: CURLE_ABORTED_BY_CALLBACK);
339 return;
340 }
341 m_ConditionVariableInit.wait(lock&: Lock, p: [this]() { return m_State != CHttpCurl::UNINITIALIZED; });
342 m_PendingRequests.emplace_back(args&: pRequestImpl);
343 curl_multi_wakeup(m: m_pMultiH);
344}
345
346bool CHttpCurl::HasIpresolveBug() const
347{
348 // curl < 7.77.0 doesn't use CURLOPT_IPRESOLVE correctly wrt.
349 // connection caches.
350 return curl_version_info(CURLVERSION_NOW)->version_num < 0x074d00;
351}
352
353void CHttpCurl::ThreadMain(void *pUser)
354{
355 static_cast<CHttpCurl *>(pUser)->RunLoop();
356}
357
358void CHttpCurl::RunLoop()
359{
360 std::unique_lock Lock(m_Lock);
361 if(curl_global_init(CURL_GLOBAL_DEFAULT))
362 {
363 log_error("http", "curl_global_init failed");
364 m_State = CHttpCurl::ERROR;
365 m_ConditionVariableInit.notify_all();
366 return;
367 }
368
369 m_pMultiH = curl_multi_init();
370 if(!m_pMultiH)
371 {
372 log_error("http", "curl_multi_init failed");
373 m_State = CHttpCurl::ERROR;
374 m_ConditionVariableInit.notify_all();
375 return;
376 }
377
378 // print curl version
379 {
380 curl_version_info_data *pVersion = curl_version_info(CURLVERSION_NOW);
381 log_info("http", "libcurl version %s (compiled = " LIBCURL_VERSION ")", pVersion->version);
382 }
383
384 m_State = CHttpCurl::RUNNING;
385 m_ConditionVariableInit.notify_all();
386 Lock.unlock();
387 m_NextTimeout = std::numeric_limits<int>::max();
388
389 while(m_State == CHttpCurl::RUNNING)
390 {
391 int Events = 0;
392 const CURLMcode PollCode = curl_multi_poll(m: m_pMultiH, extra_fds: nullptr, extra_nfds: 0, timeout_ms: m_NextTimeout, ret: &Events);
393
394 // We may have been woken up for a shutdown
395 if(m_Shutdown)
396 {
397 if(m_RunningRequests.empty() && m_PendingRequests.empty())
398 break;
399
400 auto Now = std::chrono::steady_clock::now();
401 if(!m_ShutdownTime.has_value())
402 {
403 m_ShutdownTime = Now + m_ShutdownDelay;
404 m_NextTimeout = m_ShutdownDelay.count();
405 }
406 else if(m_ShutdownTime < Now)
407 {
408 break;
409 }
410 }
411
412 if(PollCode != CURLM_OK)
413 {
414 Lock.lock();
415 log_error("http", "curl_multi_poll failed: %s", curl_multi_strerror(PollCode));
416 m_State = CHttpCurl::ERROR;
417 break;
418 }
419
420 const CURLMcode PerformCode = curl_multi_perform(m: m_pMultiH, running_handles: &Events);
421 if(PerformCode != CURLM_OK)
422 {
423 Lock.lock();
424 log_error("http", "curl_multi_perform failed: %s", curl_multi_strerror(PerformCode));
425 m_State = CHttpCurl::ERROR;
426 break;
427 }
428
429 struct CURLMsg *pMsg;
430 while((pMsg = curl_multi_info_read(m: m_pMultiH, msgs_in_queue: &Events)))
431 {
432 if(pMsg->msg == CURLMSG_DONE)
433 {
434 auto RequestIt = m_RunningRequests.find(key: pMsg->easy_handle);
435 dbg_assert(RequestIt != m_RunningRequests.end(), "Running handle not added to map");
436 auto pRequest = std::move(RequestIt->second);
437 m_RunningRequests.erase(it: RequestIt);
438
439 pRequest->OnCompletionInternal(pHandle: pMsg->easy_handle, Code: pMsg->data.result);
440 curl_multi_remove_handle(m: m_pMultiH, curl: pMsg->easy_handle);
441 curl_easy_cleanup(curl: pMsg->easy_handle);
442 }
443 }
444
445 decltype(m_PendingRequests) NewRequests = {};
446 Lock.lock();
447 std::swap(lhs&: m_PendingRequests, rhs&: NewRequests);
448 Lock.unlock();
449
450 while(!NewRequests.empty())
451 {
452 auto &pRequest = NewRequests.front();
453 if(g_Config.m_DbgHttp)
454 log_debug("http", "task: %s %s", CHttpRequestCurl::GetRequestType(pRequest->m_Type), pRequest->m_aUrl);
455
456 if(pRequest->ShouldSkipRequest())
457 {
458 if(pRequest->m_pProgressCallback != nullptr)
459 {
460 pRequest->m_pProgressCallback->OnCompletion(State: EHttpState::DONE);
461 }
462 {
463 std::unique_lock WaitLock(pRequest->m_WaitMutex);
464 pRequest->m_State = EHttpState::DONE;
465 }
466 pRequest->m_WaitCondition.notify_all();
467 NewRequests.pop_front();
468 continue;
469 }
470
471 CURL *pEH = curl_easy_init();
472 if(!pEH)
473 {
474 log_error("http", "curl_easy_init failed");
475 goto error_init;
476 }
477
478 if(!pRequest->ConfigureHandle(pHandle: pEH))
479 {
480 curl_easy_cleanup(curl: pEH);
481 str_copy(dst&: pRequest->m_aErr, src: "Failed to initialize request");
482 pRequest->OnCompletionInternal(pHandle: nullptr, Code: CURLE_ABORTED_BY_CALLBACK);
483 NewRequests.pop_front();
484 continue;
485 }
486
487 if(curl_multi_add_handle(m: m_pMultiH, curl: pEH) != CURLM_OK)
488 {
489 log_error("http", "curl_multi_add_handle failed");
490 goto error_configure;
491 }
492
493 {
494 std::unique_lock WaitLock(pRequest->m_WaitMutex);
495 pRequest->m_State = EHttpState::RUNNING;
496 }
497 m_RunningRequests.emplace(args&: pEH, args: std::move(pRequest));
498 NewRequests.pop_front();
499 continue;
500
501 error_configure:
502 curl_easy_cleanup(curl: pEH);
503 error_init:
504 Lock.lock();
505 m_State = CHttpCurl::ERROR;
506 break;
507 }
508
509 // Only happens if m_State == ERROR, thus we already hold the lock
510 if(!NewRequests.empty())
511 {
512 m_PendingRequests.insert(position: m_PendingRequests.end(), first: std::make_move_iterator(i: NewRequests.begin()), last: std::make_move_iterator(i: NewRequests.end()));
513 break;
514 }
515 }
516
517 if(!Lock.owns_lock())
518 Lock.lock();
519
520 bool Cleanup = m_State != CHttpCurl::ERROR;
521 for(auto &pRequest : m_PendingRequests)
522 {
523 str_copy(dst&: pRequest->m_aErr, src: "Shutting down");
524 pRequest->OnCompletionInternal(pHandle: nullptr, Code: CURLE_ABORTED_BY_CALLBACK);
525 }
526 m_PendingRequests.clear();
527
528 for(auto &ReqPair : m_RunningRequests)
529 {
530 auto &[pHandle, pRequest] = ReqPair;
531
532 str_copy(dst&: pRequest->m_aErr, src: "Shutting down");
533 pRequest->OnCompletionInternal(pHandle, Code: CURLE_ABORTED_BY_CALLBACK);
534
535 if(Cleanup)
536 {
537 curl_multi_remove_handle(m: m_pMultiH, curl: pHandle);
538 curl_easy_cleanup(curl: pHandle);
539 }
540 }
541 m_RunningRequests.clear();
542
543 if(Cleanup)
544 {
545 curl_multi_cleanup(m: m_pMultiH);
546 curl_global_cleanup();
547 }
548}
549
550IEngineHttp *CreateEngineHttp()
551{
552 return new CHttpCurl;
553}
554
555#endif // !CONF_PLATFORM_EMSCRIPTEN
556