1#include "serverbrowser_http.h"
2
3#include <base/lock.h>
4#include <base/log.h>
5#include <base/system.h>
6
7#include <engine/console.h>
8#include <engine/engine.h>
9#include <engine/external/json-parser/json.h>
10#include <engine/serverbrowser.h>
11#include <engine/shared/http.h>
12#include <engine/shared/jobs.h>
13#include <engine/shared/linereader.h>
14#include <engine/shared/serverinfo.h>
15#include <engine/storage.h>
16
17#include <chrono>
18#include <memory>
19#include <vector>
20
21using namespace std::chrono_literals;
22
23static int SanitizeAge(std::optional<int64_t> Age)
24{
25 // A year is of course pi*10**7 seconds.
26 if(!(Age && 0 <= *Age && *Age < 31415927))
27 {
28 return 31415927;
29 }
30 return *Age;
31}
32
33// Classify HTTP responses into buckets, treat 15 seconds as fresh, 1 minute as
34// less fresh, etc. This ensures that differences in the order of seconds do
35// not affect master choice.
36static int ClassifyAge(int AgeSeconds)
37{
38 return 0 //
39 + (AgeSeconds >= 15) // 15 seconds
40 + (AgeSeconds >= 60) // 1 minute
41 + (AgeSeconds >= 300) // 5 minutes
42 + (AgeSeconds / 3600); // 1 hour
43}
44
45class CChooseMaster
46{
47public:
48 typedef bool (*VALIDATOR)(json_value *pJson);
49
50 enum
51 {
52 MAX_URLS = 16,
53 };
54 CChooseMaster(IEngine *pEngine, IHttp *pHttp, VALIDATOR pfnValidator, const char **ppUrls, int NumUrls, int PreviousBestIndex);
55 virtual ~CChooseMaster();
56
57 bool GetBestUrl(const char **pBestUrl) const;
58 void Reset();
59 bool IsRefreshing() const { return m_pJob && !m_pJob->Done(); }
60 void Refresh();
61
62private:
63 int GetBestIndex() const;
64
65 class CData
66 {
67 public:
68 std::atomic_int m_BestIndex{-1};
69 // Constant after construction.
70 VALIDATOR m_pfnValidator;
71 int m_NumUrls;
72 char m_aaUrls[MAX_URLS][256];
73 };
74 class CJob : public IJob
75 {
76 CChooseMaster *m_pParent;
77 CLock m_Lock;
78 std::shared_ptr<CData> m_pData;
79 std::shared_ptr<CHttpRequest> m_pHead;
80 std::shared_ptr<CHttpRequest> m_pGet;
81 void Run() override REQUIRES(!m_Lock);
82
83 public:
84 CJob(CChooseMaster *pParent, std::shared_ptr<CData> pData) :
85 m_pParent(pParent),
86 m_pData(std::move(pData))
87 {
88 Abortable(Abortable: true);
89 }
90 bool Abort() override REQUIRES(!m_Lock);
91 };
92
93 IEngine *m_pEngine;
94 IHttp *m_pHttp;
95 int m_PreviousBestIndex;
96 std::shared_ptr<CData> m_pData;
97 std::shared_ptr<CJob> m_pJob;
98};
99
100CChooseMaster::CChooseMaster(IEngine *pEngine, IHttp *pHttp, VALIDATOR pfnValidator, const char **ppUrls, int NumUrls, int PreviousBestIndex) :
101 m_pEngine(pEngine),
102 m_pHttp(pHttp),
103 m_PreviousBestIndex(PreviousBestIndex)
104{
105 dbg_assert(NumUrls >= 0, "no master URLs");
106 dbg_assert(NumUrls <= MAX_URLS, "too many master URLs");
107 dbg_assert(PreviousBestIndex >= -1, "previous best index negative and not -1");
108 dbg_assert(PreviousBestIndex < NumUrls, "previous best index too high");
109 m_pData = std::make_shared<CData>();
110 m_pData->m_pfnValidator = pfnValidator;
111 m_pData->m_NumUrls = NumUrls;
112 for(int i = 0; i < m_pData->m_NumUrls; i++)
113 {
114 str_copy(dst&: m_pData->m_aaUrls[i], src: ppUrls[i]);
115 }
116}
117
118CChooseMaster::~CChooseMaster()
119{
120 if(m_pJob)
121 {
122 m_pJob->Abort();
123 }
124}
125
126int CChooseMaster::GetBestIndex() const
127{
128 int BestIndex = m_pData->m_BestIndex.load();
129 if(BestIndex >= 0)
130 {
131 return BestIndex;
132 }
133 else
134 {
135 return m_PreviousBestIndex;
136 }
137}
138
139bool CChooseMaster::GetBestUrl(const char **ppBestUrl) const
140{
141 int Index = GetBestIndex();
142 if(Index < 0)
143 {
144 *ppBestUrl = nullptr;
145 return true;
146 }
147 *ppBestUrl = m_pData->m_aaUrls[Index];
148 return false;
149}
150
151void CChooseMaster::Reset()
152{
153 m_PreviousBestIndex = -1;
154 m_pData->m_BestIndex.store(i: -1);
155}
156
157void CChooseMaster::Refresh()
158{
159 if(m_pJob == nullptr || m_pJob->State() == IJob::STATE_DONE)
160 {
161 m_pJob = std::make_shared<CJob>(args: this, args&: m_pData);
162 m_pEngine->AddJob(pJob: m_pJob);
163 }
164}
165
166bool CChooseMaster::CJob::Abort()
167{
168 if(!IJob::Abort())
169 {
170 return false;
171 }
172
173 const CLockScope LockScope(m_Lock);
174 if(m_pHead != nullptr)
175 {
176 m_pHead->Abort();
177 }
178
179 if(m_pGet != nullptr)
180 {
181 m_pGet->Abort();
182 }
183
184 return true;
185}
186
187void CChooseMaster::CJob::Run()
188{
189 // Check masters in a random order.
190 int aRandomized[MAX_URLS] = {0};
191 for(int i = 0; i < m_pData->m_NumUrls; i++)
192 {
193 aRandomized[i] = i;
194 }
195 // https://en.wikipedia.org/w/index.php?title=Fisher%E2%80%93Yates_shuffle&oldid=1002922479#The_modern_algorithm
196 // The equivalent version.
197 for(int i = 0; i <= m_pData->m_NumUrls - 2; i++)
198 {
199 int j = i + secure_rand_below(below: m_pData->m_NumUrls - i);
200 std::swap(a&: aRandomized[i], b&: aRandomized[j]);
201 }
202 // Do a HEAD request to ensure that a connection is established and
203 // then do a GET request to check how fast we can get the server list.
204 //
205 // 10 seconds connection timeout, lower than 8KB/s for 10 seconds to
206 // fail.
207 CTimeout Timeout{.m_ConnectTimeoutMs: 10000, .m_TimeoutMs: 0, .m_LowSpeedLimit: 8000, .m_LowSpeedTime: 10};
208 int aTimeMs[MAX_URLS];
209 int aAgeS[MAX_URLS];
210 for(int i = 0; i < m_pData->m_NumUrls; i++)
211 {
212 aTimeMs[i] = -1;
213 aAgeS[i] = SanitizeAge(Age: {});
214 const char *pUrl = m_pData->m_aaUrls[aRandomized[i]];
215 std::shared_ptr<CHttpRequest> pHead = HttpHead(pUrl);
216 pHead->Timeout(Timeout);
217 pHead->LogProgress(LogProgress: HTTPLOG::FAILURE);
218 {
219 const CLockScope LockScope(m_Lock);
220 m_pHead = pHead;
221 }
222
223 m_pParent->m_pHttp->Run(pRequest: pHead);
224 pHead->Wait();
225 if(pHead->State() == EHttpState::ABORTED || State() == IJob::STATE_ABORTED)
226 {
227 log_debug("serverbrowser_http", "master chooser aborted");
228 return;
229 }
230 if(pHead->State() != EHttpState::DONE)
231 {
232 continue;
233 }
234
235 auto StartTime = time_get_nanoseconds();
236 std::shared_ptr<CHttpRequest> pGet = HttpGet(pUrl);
237 pGet->Timeout(Timeout);
238 pGet->LogProgress(LogProgress: HTTPLOG::FAILURE);
239 {
240 const CLockScope LockScope(m_Lock);
241 m_pGet = pGet;
242 }
243
244 m_pParent->m_pHttp->Run(pRequest: pGet);
245 pGet->Wait();
246
247 auto Time = std::chrono::duration_cast<std::chrono::milliseconds>(d: time_get_nanoseconds() - StartTime);
248 if(pGet->State() == EHttpState::ABORTED || State() == IJob::STATE_ABORTED)
249 {
250 log_debug("serverbrowser_http", "master chooser aborted");
251 return;
252 }
253 if(pGet->State() != EHttpState::DONE)
254 {
255 continue;
256 }
257 json_value *pJson = pGet->ResultJson();
258 if(!pJson)
259 {
260 continue;
261 }
262
263 bool ParseFailure = m_pData->m_pfnValidator(pJson);
264 json_value_free(pJson);
265 if(ParseFailure)
266 {
267 continue;
268 }
269 int AgeS = SanitizeAge(Age: pGet->ResultAgeSeconds());
270 log_info("serverbrowser_http", "found master, url='%s' time=%dms age=%ds", pUrl, (int)Time.count(), AgeS);
271
272 aTimeMs[i] = Time.count();
273 aAgeS[i] = AgeS;
274 }
275
276 // Determine index of the minimum time.
277 int BestIndex = -1;
278 int BestTime = 0;
279 int BestAge = 0;
280 for(int i = 0; i < m_pData->m_NumUrls; i++)
281 {
282 if(aTimeMs[i] < 0)
283 {
284 continue;
285 }
286 if(BestIndex == -1 || std::tuple(ClassifyAge(AgeSeconds: aAgeS[i]), aTimeMs[i]) < std::tuple(ClassifyAge(AgeSeconds: BestAge), BestTime))
287 {
288 BestTime = aTimeMs[i];
289 BestAge = aAgeS[i];
290 BestIndex = aRandomized[i];
291 }
292 }
293 if(BestIndex == -1)
294 {
295 log_error("serverbrowser_http", "WARNING: no usable masters found");
296 return;
297 }
298
299 log_info("serverbrowser_http", "determined best master, url='%s' time=%dms age=%ds", m_pData->m_aaUrls[BestIndex], BestTime, BestAge);
300 m_pData->m_BestIndex.store(i: BestIndex);
301}
302
303class CServerBrowserHttp : public IServerBrowserHttp
304{
305public:
306 CServerBrowserHttp(IEngine *pEngine, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex);
307 ~CServerBrowserHttp() override;
308 void Update() override;
309 bool IsRefreshing() const override { return m_State != STATE_DONE && m_State != STATE_NO_MASTER; }
310 bool IsError() const override { return m_State == STATE_NO_MASTER; }
311 void Refresh() override;
312 bool GetBestUrl(const char **pBestUrl) const override { return m_pChooseMaster->GetBestUrl(ppBestUrl: pBestUrl); }
313
314 int NumServers() const override
315 {
316 return m_vServers.size();
317 }
318 const CServerInfo &Server(int Index) const override
319 {
320 return m_vServers[Index];
321 }
322
323private:
324 enum
325 {
326 STATE_DONE,
327 STATE_WANTREFRESH,
328 STATE_REFRESHING,
329 STATE_NO_MASTER,
330 };
331
332 static bool Validate(json_value *pJson);
333 static bool Parse(json_value *pJson, std::vector<CServerInfo> *pvServers);
334
335 IHttp *m_pHttp;
336
337 int m_State = STATE_WANTREFRESH;
338 std::shared_ptr<CHttpRequest> m_pGetServers;
339 std::unique_ptr<CChooseMaster> m_pChooseMaster;
340
341 std::vector<CServerInfo> m_vServers;
342};
343
344CServerBrowserHttp::CServerBrowserHttp(IEngine *pEngine, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex) :
345 m_pHttp(pHttp),
346 m_pChooseMaster(new CChooseMaster(pEngine, pHttp, Validate, ppUrls, NumUrls, PreviousBestIndex))
347{
348 Refresh();
349}
350
351CServerBrowserHttp::~CServerBrowserHttp()
352{
353 if(m_pGetServers != nullptr)
354 {
355 m_pGetServers->Abort();
356 }
357}
358
359void CServerBrowserHttp::Update()
360{
361 if(m_State == STATE_WANTREFRESH)
362 {
363 const char *pBestUrl;
364 if(m_pChooseMaster->GetBestUrl(ppBestUrl: &pBestUrl))
365 {
366 if(!m_pChooseMaster->IsRefreshing())
367 {
368 log_error("serverbrowser_http", "no working serverlist URL found");
369 m_State = STATE_NO_MASTER;
370 }
371 return;
372 }
373 m_pGetServers = HttpGet(pUrl: pBestUrl);
374 // 10 seconds connection timeout, lower than 8KB/s for 10 seconds to fail.
375 m_pGetServers->Timeout(Timeout: CTimeout{.m_ConnectTimeoutMs: 10000, .m_TimeoutMs: 0, .m_LowSpeedLimit: 8000, .m_LowSpeedTime: 10});
376 m_pHttp->Run(pRequest: m_pGetServers);
377 m_State = STATE_REFRESHING;
378 }
379 else if(m_State == STATE_REFRESHING)
380 {
381 if(!m_pGetServers->Done())
382 {
383 return;
384 }
385 m_State = STATE_DONE;
386 std::shared_ptr<CHttpRequest> pGetServers = nullptr;
387 std::swap(a&: m_pGetServers, b&: pGetServers);
388
389 bool Success = true;
390 json_value *pJson = pGetServers->State() == EHttpState::DONE ? pGetServers->ResultJson() : nullptr;
391 Success = Success && pJson;
392 Success = Success && !Parse(pJson, pvServers: &m_vServers);
393 json_value_free(pJson);
394 if(!Success)
395 {
396 log_error("serverbrowser_http", "failed getting serverlist, trying to find best URL");
397 m_pChooseMaster->Reset();
398 m_pChooseMaster->Refresh();
399 }
400 else
401 {
402 // Try to find new master if the current one returns
403 // results that are 5 minutes old.
404 int Age = SanitizeAge(Age: pGetServers->ResultAgeSeconds());
405 if(Age > 300)
406 {
407 log_info("serverbrowser_http", "got stale serverlist, age=%ds, trying to find best URL", Age);
408 m_pChooseMaster->Refresh();
409 }
410 }
411 }
412}
413void CServerBrowserHttp::Refresh()
414{
415 if(m_State == STATE_WANTREFRESH || m_State == STATE_REFRESHING || m_State == STATE_NO_MASTER)
416 {
417 if(m_State == STATE_NO_MASTER)
418 m_State = STATE_WANTREFRESH;
419 m_pChooseMaster->Refresh();
420 }
421 if(m_State == STATE_DONE)
422 m_State = STATE_WANTREFRESH;
423 Update();
424}
425static bool ServerbrowserParseUrl(NETADDR *pOut, const char *pUrl)
426{
427 int Failure = net_addr_from_url(addr: pOut, string: pUrl, host_buf: nullptr, host_buf_size: 0);
428 if(Failure || pOut->port == 0)
429 return true;
430 return false;
431}
432bool CServerBrowserHttp::Validate(json_value *pJson)
433{
434 std::vector<CServerInfo> vServers;
435 return Parse(pJson, pvServers: &vServers);
436}
437bool CServerBrowserHttp::Parse(json_value *pJson, std::vector<CServerInfo> *pvServers)
438{
439 std::vector<CServerInfo> vServers;
440
441 const json_value &Json = *pJson;
442 const json_value &Servers = Json["servers"];
443 if(Servers.type != json_array)
444 {
445 return true;
446 }
447 for(unsigned int i = 0; i < Servers.u.array.length; i++)
448 {
449 const json_value &Server = Servers[i];
450 const json_value &Addresses = Server["addresses"];
451 const json_value &Info = Server["info"];
452 const json_value &Location = Server["location"];
453 int ParsedLocation = CServerInfo::LOC_UNKNOWN;
454 CServerInfo2 ParsedInfo;
455 if(Addresses.type != json_array || (Location.type != json_string && Location.type != json_none))
456 {
457 return true;
458 }
459 if(Location.type == json_string)
460 {
461 if(CServerInfo::ParseLocation(pResult: &ParsedLocation, pString: Location))
462 {
463 return true;
464 }
465 }
466 if(CServerInfo2::FromJson(pOut: &ParsedInfo, pJson: &Info))
467 {
468 // Only skip the current server on parsing
469 // failure; the server info is "user input" by
470 // the game server and can be set to arbitrary
471 // values.
472 continue;
473 }
474 CServerInfo SetInfo = ParsedInfo;
475 SetInfo.m_Location = ParsedLocation;
476 SetInfo.m_NumAddresses = 0;
477 bool GotVersion6 = false;
478 for(unsigned int a = 0; a < Addresses.u.array.length; a++)
479 {
480 const json_value &Address = Addresses[a];
481 if(Address.type != json_string)
482 {
483 return true;
484 }
485 if(str_startswith(str: Addresses[a], prefix: "tw-0.6+udp://"))
486 {
487 GotVersion6 = true;
488 break;
489 }
490 }
491 for(unsigned int a = 0; a < Addresses.u.array.length; a++)
492 {
493 const json_value &Address = Addresses[a];
494 if(Address.type != json_string)
495 {
496 return true;
497 }
498 if(GotVersion6 && str_startswith(str: Addresses[a], prefix: "tw-0.7+udp://"))
499 {
500 continue;
501 }
502 NETADDR ParsedAddr;
503 if(ServerbrowserParseUrl(pOut: &ParsedAddr, pUrl: Addresses[a]))
504 {
505 // Skip unknown addresses.
506 continue;
507 }
508 if(SetInfo.m_NumAddresses < (int)std::size(SetInfo.m_aAddresses))
509 {
510 SetInfo.m_aAddresses[SetInfo.m_NumAddresses] = ParsedAddr;
511 SetInfo.m_NumAddresses += 1;
512 }
513 }
514 if(SetInfo.m_NumAddresses > 0)
515 {
516 vServers.push_back(x: SetInfo);
517 }
518 }
519 *pvServers = vServers;
520 return false;
521}
522
523static const char *DEFAULT_SERVERLIST_URLS[] = {
524 "https://master1.ddnet.org/ddnet/15/servers.json",
525 "https://master2.ddnet.org/ddnet/15/servers.json",
526 "https://master3.ddnet.org/ddnet/15/servers.json",
527 "https://master4.ddnet.org/ddnet/15/servers.json",
528};
529
530IServerBrowserHttp *CreateServerBrowserHttp(IEngine *pEngine, IStorage *pStorage, IHttp *pHttp, const char *pPreviousBestUrl)
531{
532 char aaUrls[CChooseMaster::MAX_URLS][256];
533 const char *apUrls[CChooseMaster::MAX_URLS] = {nullptr};
534 const char **ppUrls = apUrls;
535 int NumUrls = 0;
536 CLineReader LineReader;
537 if(LineReader.OpenFile(File: pStorage->OpenFile(pFilename: "ddnet-serverlist-urls.cfg", Flags: IOFLAG_READ, Type: IStorage::TYPE_ALL)))
538 {
539 while(const char *pLine = LineReader.Get())
540 {
541 if(NumUrls == CChooseMaster::MAX_URLS)
542 {
543 break;
544 }
545 str_copy(dst&: aaUrls[NumUrls], src: pLine);
546 apUrls[NumUrls] = aaUrls[NumUrls];
547 NumUrls += 1;
548 }
549 }
550 if(NumUrls == 0)
551 {
552 ppUrls = DEFAULT_SERVERLIST_URLS;
553 NumUrls = std::size(DEFAULT_SERVERLIST_URLS);
554 }
555 int PreviousBestIndex = -1;
556 for(int i = 0; i < NumUrls; i++)
557 {
558 if(str_comp(a: ppUrls[i], b: pPreviousBestUrl) == 0)
559 {
560 PreviousBestIndex = i;
561 break;
562 }
563 }
564 return new CServerBrowserHttp(pEngine, pHttp, ppUrls, NumUrls, PreviousBestIndex);
565}
566