1#include "serverbrowser_http.h"
2
3#include <base/dbg.h>
4#include <base/io.h>
5#include <base/lock.h>
6#include <base/log.h>
7#include <base/net.h>
8#include <base/secure.h>
9#include <base/str.h>
10#include <base/time.h>
11
12#include <engine/console.h>
13#include <engine/engine.h>
14#include <engine/external/json-parser/json.h>
15#include <engine/serverbrowser.h>
16#include <engine/shared/http.h>
17#include <engine/shared/jobs.h>
18#include <engine/shared/linereader.h>
19#include <engine/shared/serverinfo.h>
20#include <engine/storage.h>
21
22#include <chrono>
23#include <memory>
24#include <vector>
25
26using namespace std::chrono_literals;
27
28static int SanitizeAge(std::optional<int64_t> Age)
29{
30 // A year is of course pi*10**7 seconds.
31 if(!(Age && 0 <= *Age && *Age < 31415927))
32 {
33 return 31415927;
34 }
35 return *Age;
36}
37
38// Classify HTTP responses into buckets, treat 15 seconds as fresh, 1 minute as
39// less fresh, etc. This ensures that differences in the order of seconds do
40// not affect master choice.
41static int ClassifyAge(int AgeSeconds)
42{
43 return 0 //
44 + (AgeSeconds >= 15) // 15 seconds
45 + (AgeSeconds >= 60) // 1 minute
46 + (AgeSeconds >= 300) // 5 minutes
47 + (AgeSeconds / 3600); // 1 hour
48}
49
50class CChooseMaster
51{
52public:
53 typedef bool (*VALIDATOR)(json_value *pJson);
54
55 enum
56 {
57 MAX_URLS = 16,
58 };
59 CChooseMaster(IEngine *pEngine, IHttp *pHttp, VALIDATOR pfnValidator, const char **ppUrls, int NumUrls, int PreviousBestIndex);
60 virtual ~CChooseMaster();
61
62 bool GetBestUrl(const char **pBestUrl) const;
63 void Reset();
64 bool IsRefreshing() const { return m_pJob && !m_pJob->Done(); }
65 void Refresh();
66
67private:
68 int GetBestIndex() const;
69
70 class CData
71 {
72 public:
73 std::atomic_int m_BestIndex{-1};
74 // Constant after construction.
75 VALIDATOR m_pfnValidator;
76 int m_NumUrls;
77 char m_aaUrls[MAX_URLS][256];
78 };
79 class CJob : public IJob
80 {
81 CChooseMaster *m_pParent;
82 CLock m_Lock;
83 std::shared_ptr<CData> m_pData;
84 std::shared_ptr<CHttpRequest> m_pHead;
85 std::shared_ptr<CHttpRequest> m_pGet;
86 void Run() override REQUIRES(!m_Lock);
87
88 public:
89 CJob(CChooseMaster *pParent, std::shared_ptr<CData> pData) :
90 m_pParent(pParent),
91 m_pData(std::move(pData))
92 {
93 Abortable(Abortable: true);
94 }
95 bool Abort() override REQUIRES(!m_Lock);
96 };
97
98 IEngine *m_pEngine;
99 IHttp *m_pHttp;
100 int m_PreviousBestIndex;
101 std::shared_ptr<CData> m_pData;
102 std::shared_ptr<CJob> m_pJob;
103};
104
105CChooseMaster::CChooseMaster(IEngine *pEngine, IHttp *pHttp, VALIDATOR pfnValidator, const char **ppUrls, int NumUrls, int PreviousBestIndex) :
106 m_pEngine(pEngine),
107 m_pHttp(pHttp),
108 m_PreviousBestIndex(PreviousBestIndex)
109{
110 dbg_assert(NumUrls >= 0, "no master URLs");
111 dbg_assert(NumUrls <= MAX_URLS, "too many master URLs");
112 dbg_assert(PreviousBestIndex >= -1, "previous best index negative and not -1");
113 dbg_assert(PreviousBestIndex < NumUrls, "previous best index too high");
114 m_pData = std::make_shared<CData>();
115 m_pData->m_pfnValidator = pfnValidator;
116 m_pData->m_NumUrls = NumUrls;
117 for(int i = 0; i < m_pData->m_NumUrls; i++)
118 {
119 str_copy(dst&: m_pData->m_aaUrls[i], src: ppUrls[i]);
120 }
121}
122
123CChooseMaster::~CChooseMaster()
124{
125 if(m_pJob)
126 {
127 m_pJob->Abort();
128 }
129}
130
131int CChooseMaster::GetBestIndex() const
132{
133 int BestIndex = m_pData->m_BestIndex.load();
134 if(BestIndex >= 0)
135 {
136 return BestIndex;
137 }
138 else
139 {
140 return m_PreviousBestIndex;
141 }
142}
143
144bool CChooseMaster::GetBestUrl(const char **ppBestUrl) const
145{
146 int Index = GetBestIndex();
147 if(Index < 0)
148 {
149 *ppBestUrl = nullptr;
150 return true;
151 }
152 *ppBestUrl = m_pData->m_aaUrls[Index];
153 return false;
154}
155
156void CChooseMaster::Reset()
157{
158 m_PreviousBestIndex = -1;
159 m_pData->m_BestIndex.store(i: -1);
160}
161
162void CChooseMaster::Refresh()
163{
164 if(m_pJob == nullptr || m_pJob->State() == IJob::STATE_DONE)
165 {
166 m_pJob = std::make_shared<CJob>(args: this, args&: m_pData);
167 m_pEngine->AddJob(pJob: m_pJob);
168 }
169}
170
171bool CChooseMaster::CJob::Abort()
172{
173 if(!IJob::Abort())
174 {
175 return false;
176 }
177
178 const CLockScope LockScope(m_Lock);
179 if(m_pHead != nullptr)
180 {
181 m_pHead->Abort();
182 }
183
184 if(m_pGet != nullptr)
185 {
186 m_pGet->Abort();
187 }
188
189 return true;
190}
191
192void CChooseMaster::CJob::Run()
193{
194 // Check masters in a random order.
195 int aRandomized[MAX_URLS] = {0};
196 for(int i = 0; i < m_pData->m_NumUrls; i++)
197 {
198 aRandomized[i] = i;
199 }
200 // https://en.wikipedia.org/w/index.php?title=Fisher%E2%80%93Yates_shuffle&oldid=1002922479#The_modern_algorithm
201 // The equivalent version.
202 for(int i = 0; i <= m_pData->m_NumUrls - 2; i++)
203 {
204 int j = i + secure_rand_below(below: m_pData->m_NumUrls - i);
205 std::swap(a&: aRandomized[i], b&: aRandomized[j]);
206 }
207 // Do a HEAD request to ensure that a connection is established and
208 // then do a GET request to check how fast we can get the server list.
209 //
210 // 10 seconds connection timeout, lower than 8KB/s for 10 seconds to
211 // fail.
212 CTimeout Timeout{.m_ConnectTimeoutMs: 10000, .m_TimeoutMs: 0, .m_LowSpeedLimit: 8000, .m_LowSpeedTime: 10};
213 int aTimeMs[MAX_URLS];
214 int aAgeS[MAX_URLS];
215 for(int i = 0; i < m_pData->m_NumUrls; i++)
216 {
217 aTimeMs[i] = -1;
218 aAgeS[i] = SanitizeAge(Age: {});
219 const char *pUrl = m_pData->m_aaUrls[aRandomized[i]];
220 std::shared_ptr<CHttpRequest> pHead = HttpHead(pUrl);
221 pHead->Timeout(Timeout);
222 pHead->LogProgress(LogProgress: HTTPLOG::FAILURE);
223 {
224 const CLockScope LockScope(m_Lock);
225 m_pHead = pHead;
226 }
227
228 m_pParent->m_pHttp->Run(pRequest: pHead);
229 pHead->Wait();
230 if(pHead->State() == EHttpState::ABORTED || State() == IJob::STATE_ABORTED)
231 {
232 log_debug("serverbrowser_http", "master chooser aborted");
233 return;
234 }
235 if(pHead->State() != EHttpState::DONE)
236 {
237 continue;
238 }
239
240 auto StartTime = time_get_nanoseconds();
241 std::shared_ptr<CHttpRequest> pGet = HttpGet(pUrl);
242 pGet->Timeout(Timeout);
243 pGet->LogProgress(LogProgress: HTTPLOG::FAILURE);
244 {
245 const CLockScope LockScope(m_Lock);
246 m_pGet = pGet;
247 }
248
249 m_pParent->m_pHttp->Run(pRequest: pGet);
250 pGet->Wait();
251
252 auto Time = std::chrono::duration_cast<std::chrono::milliseconds>(d: time_get_nanoseconds() - StartTime);
253 if(pGet->State() == EHttpState::ABORTED || State() == IJob::STATE_ABORTED)
254 {
255 log_debug("serverbrowser_http", "master chooser aborted");
256 return;
257 }
258 if(pGet->State() != EHttpState::DONE)
259 {
260 continue;
261 }
262 json_value *pJson = pGet->ResultJson();
263 if(!pJson)
264 {
265 continue;
266 }
267
268 bool ParseFailure = m_pData->m_pfnValidator(pJson);
269 json_value_free(pJson);
270 if(ParseFailure)
271 {
272 continue;
273 }
274 int AgeS = SanitizeAge(Age: pGet->ResultAgeSeconds());
275 log_info("serverbrowser_http", "found master, url='%s' time=%dms age=%ds", pUrl, (int)Time.count(), AgeS);
276
277 aTimeMs[i] = Time.count();
278 aAgeS[i] = AgeS;
279 }
280
281 // Determine index of the minimum time.
282 int BestIndex = -1;
283 int BestTime = 0;
284 int BestAge = 0;
285 for(int i = 0; i < m_pData->m_NumUrls; i++)
286 {
287 if(aTimeMs[i] < 0)
288 {
289 continue;
290 }
291 if(BestIndex == -1 || std::tuple(ClassifyAge(AgeSeconds: aAgeS[i]), aTimeMs[i]) < std::tuple(ClassifyAge(AgeSeconds: BestAge), BestTime))
292 {
293 BestTime = aTimeMs[i];
294 BestAge = aAgeS[i];
295 BestIndex = aRandomized[i];
296 }
297 }
298 if(BestIndex == -1)
299 {
300 log_error("serverbrowser_http", "WARNING: no usable masters found");
301 return;
302 }
303
304 log_info("serverbrowser_http", "determined best master, url='%s' time=%dms age=%ds", m_pData->m_aaUrls[BestIndex], BestTime, BestAge);
305 m_pData->m_BestIndex.store(i: BestIndex);
306}
307
308class CServerBrowserHttp : public IServerBrowserHttp
309{
310public:
311 CServerBrowserHttp(IEngine *pEngine, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex);
312 ~CServerBrowserHttp() override;
313 void Update() override;
314 bool IsRefreshing() const override { return m_State != STATE_DONE && m_State != STATE_NO_MASTER; }
315 bool IsError() const override { return m_State == STATE_NO_MASTER; }
316 void Refresh() override;
317 bool GetBestUrl(const char **pBestUrl) const override { return m_pChooseMaster->GetBestUrl(ppBestUrl: pBestUrl); }
318
319 int NumServers() const override
320 {
321 return m_vServers.size();
322 }
323 const CServerInfo &Server(int Index) const override
324 {
325 return m_vServers[Index];
326 }
327
328private:
329 enum
330 {
331 STATE_DONE,
332 STATE_WANTREFRESH,
333 STATE_REFRESHING,
334 STATE_NO_MASTER,
335 };
336
337 static bool Validate(json_value *pJson);
338 static bool Parse(json_value *pJson, std::vector<CServerInfo> *pvServers);
339
340 IHttp *m_pHttp;
341
342 int m_State = STATE_WANTREFRESH;
343 std::shared_ptr<CHttpRequest> m_pGetServers;
344 std::unique_ptr<CChooseMaster> m_pChooseMaster;
345
346 std::vector<CServerInfo> m_vServers;
347};
348
349CServerBrowserHttp::CServerBrowserHttp(IEngine *pEngine, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex) :
350 m_pHttp(pHttp),
351 m_pChooseMaster(new CChooseMaster(pEngine, pHttp, Validate, ppUrls, NumUrls, PreviousBestIndex))
352{
353 Refresh();
354}
355
356CServerBrowserHttp::~CServerBrowserHttp()
357{
358 if(m_pGetServers != nullptr)
359 {
360 m_pGetServers->Abort();
361 }
362}
363
364void CServerBrowserHttp::Update()
365{
366 if(m_State == STATE_WANTREFRESH)
367 {
368 const char *pBestUrl;
369 if(m_pChooseMaster->GetBestUrl(ppBestUrl: &pBestUrl))
370 {
371 if(!m_pChooseMaster->IsRefreshing())
372 {
373 log_error("serverbrowser_http", "no working serverlist URL found");
374 m_State = STATE_NO_MASTER;
375 }
376 return;
377 }
378 m_pGetServers = HttpGet(pUrl: pBestUrl);
379 // 10 seconds connection timeout, lower than 8KB/s for 10 seconds to fail.
380 m_pGetServers->Timeout(Timeout: CTimeout{.m_ConnectTimeoutMs: 10000, .m_TimeoutMs: 0, .m_LowSpeedLimit: 8000, .m_LowSpeedTime: 10});
381 m_pHttp->Run(pRequest: m_pGetServers);
382 m_State = STATE_REFRESHING;
383 }
384 else if(m_State == STATE_REFRESHING)
385 {
386 if(!m_pGetServers->Done())
387 {
388 return;
389 }
390 m_State = STATE_DONE;
391 std::shared_ptr<CHttpRequest> pGetServers = nullptr;
392 std::swap(a&: m_pGetServers, b&: pGetServers);
393
394 bool Success = true;
395 json_value *pJson = pGetServers->State() == EHttpState::DONE ? pGetServers->ResultJson() : nullptr;
396 Success = Success && pJson;
397 Success = Success && !Parse(pJson, pvServers: &m_vServers);
398 json_value_free(pJson);
399 if(!Success)
400 {
401 log_error("serverbrowser_http", "failed getting serverlist, trying to find best URL");
402 m_pChooseMaster->Reset();
403 m_pChooseMaster->Refresh();
404 }
405 else
406 {
407 // Try to find new master if the current one returns
408 // results that are 5 minutes old.
409 int Age = SanitizeAge(Age: pGetServers->ResultAgeSeconds());
410 if(Age > 300)
411 {
412 log_info("serverbrowser_http", "got stale serverlist, age=%ds, trying to find best URL", Age);
413 m_pChooseMaster->Refresh();
414 }
415 }
416 }
417}
418void CServerBrowserHttp::Refresh()
419{
420 if(m_State == STATE_WANTREFRESH || m_State == STATE_REFRESHING || m_State == STATE_NO_MASTER)
421 {
422 if(m_State == STATE_NO_MASTER)
423 m_State = STATE_WANTREFRESH;
424 m_pChooseMaster->Refresh();
425 }
426 if(m_State == STATE_DONE)
427 m_State = STATE_WANTREFRESH;
428 Update();
429}
430static bool ServerbrowserParseUrl(NETADDR *pOut, const char *pUrl)
431{
432 int Failure = net_addr_from_url(addr: pOut, string: pUrl, host_buf: nullptr, host_buf_size: 0);
433 if(Failure || pOut->port == 0)
434 return true;
435 return false;
436}
437bool CServerBrowserHttp::Validate(json_value *pJson)
438{
439 std::vector<CServerInfo> vServers;
440 return Parse(pJson, pvServers: &vServers);
441}
442bool CServerBrowserHttp::Parse(json_value *pJson, std::vector<CServerInfo> *pvServers)
443{
444 std::vector<CServerInfo> vServers;
445
446 const json_value &Json = *pJson;
447 const json_value &Servers = Json["servers"];
448 if(Servers.type != json_array)
449 {
450 return true;
451 }
452 for(unsigned int i = 0; i < Servers.u.array.length; i++)
453 {
454 const json_value &Server = Servers[i];
455 const json_value &Addresses = Server["addresses"];
456 const json_value &Info = Server["info"];
457 const json_value &Location = Server["location"];
458 int ParsedLocation = CServerInfo::LOC_UNKNOWN;
459 CServerInfo2 ParsedInfo;
460 if(Addresses.type != json_array || (Location.type != json_string && Location.type != json_none))
461 {
462 return true;
463 }
464 if(Location.type == json_string)
465 {
466 if(CServerInfo::ParseLocation(pResult: &ParsedLocation, pString: Location))
467 {
468 return true;
469 }
470 }
471 if(CServerInfo2::FromJson(pOut: &ParsedInfo, pJson: &Info))
472 {
473 // Only skip the current server on parsing
474 // failure; the server info is "user input" by
475 // the game server and can be set to arbitrary
476 // values.
477 continue;
478 }
479 CServerInfo SetInfo = ParsedInfo;
480 SetInfo.m_Location = ParsedLocation;
481 SetInfo.m_NumAddresses = 0;
482 bool GotVersion6 = false;
483 for(unsigned int a = 0; a < Addresses.u.array.length; a++)
484 {
485 const json_value &Address = Addresses[a];
486 if(Address.type != json_string)
487 {
488 return true;
489 }
490 if(str_startswith(str: Addresses[a], prefix: "tw-0.6+udp://"))
491 {
492 GotVersion6 = true;
493 break;
494 }
495 }
496 for(unsigned int a = 0; a < Addresses.u.array.length; a++)
497 {
498 const json_value &Address = Addresses[a];
499 if(Address.type != json_string)
500 {
501 return true;
502 }
503 if(GotVersion6 && str_startswith(str: Addresses[a], prefix: "tw-0.7+udp://"))
504 {
505 continue;
506 }
507 NETADDR ParsedAddr;
508 if(ServerbrowserParseUrl(pOut: &ParsedAddr, pUrl: Addresses[a]))
509 {
510 // Skip unknown addresses.
511 continue;
512 }
513 if(SetInfo.m_NumAddresses < (int)std::size(SetInfo.m_aAddresses))
514 {
515 SetInfo.m_aAddresses[SetInfo.m_NumAddresses] = ParsedAddr;
516 SetInfo.m_NumAddresses += 1;
517 }
518 }
519 if(SetInfo.m_NumAddresses > 0)
520 {
521 vServers.push_back(x: SetInfo);
522 }
523 }
524 *pvServers = vServers;
525 return false;
526}
527
528static const char *DEFAULT_SERVERLIST_URLS[] = {
529 "https://master1.ddnet.org/ddnet/15/servers.json",
530 "https://master2.ddnet.org/ddnet/15/servers.json",
531 "https://master3.ddnet.org/ddnet/15/servers.json",
532 "https://master4.ddnet.org/ddnet/15/servers.json",
533};
534
535IServerBrowserHttp *CreateServerBrowserHttp(IEngine *pEngine, IStorage *pStorage, IHttp *pHttp, const char *pPreviousBestUrl)
536{
537 char aaUrls[CChooseMaster::MAX_URLS][256];
538 const char *apUrls[CChooseMaster::MAX_URLS] = {nullptr};
539 const char **ppUrls = apUrls;
540 int NumUrls = 0;
541 CLineReader LineReader;
542 if(LineReader.OpenFile(File: pStorage->OpenFile(pFilename: "ddnet-serverlist-urls.cfg", Flags: IOFLAG_READ, Type: IStorage::TYPE_ALL)))
543 {
544 while(const char *pLine = LineReader.Get())
545 {
546 if(NumUrls == CChooseMaster::MAX_URLS)
547 {
548 break;
549 }
550 str_copy(dst&: aaUrls[NumUrls], src: pLine);
551 apUrls[NumUrls] = aaUrls[NumUrls];
552 NumUrls += 1;
553 }
554 }
555 if(NumUrls == 0)
556 {
557 ppUrls = DEFAULT_SERVERLIST_URLS;
558 NumUrls = std::size(DEFAULT_SERVERLIST_URLS);
559 }
560 int PreviousBestIndex = -1;
561 for(int i = 0; i < NumUrls; i++)
562 {
563 if(str_comp(a: ppUrls[i], b: pPreviousBestUrl) == 0)
564 {
565 PreviousBestIndex = i;
566 break;
567 }
568 }
569 return new CServerBrowserHttp(pEngine, pHttp, ppUrls, NumUrls, PreviousBestIndex);
570}
571