1#include "serverbrowser_http.h"
2
3#include <engine/console.h>
4#include <engine/engine.h>
5#include <engine/external/json-parser/json.h>
6#include <engine/serverbrowser.h>
7#include <engine/shared/http.h>
8#include <engine/shared/jobs.h>
9#include <engine/shared/linereader.h>
10#include <engine/shared/serverinfo.h>
11#include <engine/storage.h>
12
13#include <base/lock.h>
14#include <base/log.h>
15#include <base/system.h>
16
17#include <memory>
18#include <vector>
19
20#include <chrono>
21
22using namespace std::chrono_literals;
23
24static int SanitizeAge(std::optional<int64_t> Age)
25{
26 // A year is of course pi*10**7 seconds.
27 if(!(Age && 0 <= *Age && *Age < 31415927))
28 {
29 return 31415927;
30 }
31 return *Age;
32}
33
34// Classify HTTP responses into buckets, treat 15 seconds as fresh, 1 minute as
35// less fresh, etc. This ensures that differences in the order of seconds do
36// not affect master choice.
37static int ClassifyAge(int AgeSeconds)
38{
39 return 0 //
40 + (AgeSeconds >= 15) // 15 seconds
41 + (AgeSeconds >= 60) // 1 minute
42 + (AgeSeconds >= 300) // 5 minutes
43 + (AgeSeconds / 3600); // 1 hour
44}
45
46class CChooseMaster
47{
48public:
49 typedef bool (*VALIDATOR)(json_value *pJson);
50
51 enum
52 {
53 MAX_URLS = 16,
54 };
55 CChooseMaster(IEngine *pEngine, IHttp *pHttp, VALIDATOR pfnValidator, const char **ppUrls, int NumUrls, int PreviousBestIndex);
56 virtual ~CChooseMaster();
57
58 bool GetBestUrl(const char **pBestUrl) const;
59 void Reset();
60 bool IsRefreshing() const { return m_pJob && !m_pJob->Done(); }
61 void Refresh();
62
63private:
64 int GetBestIndex() const;
65
66 class CData
67 {
68 public:
69 std::atomic_int m_BestIndex{-1};
70 // Constant after construction.
71 VALIDATOR m_pfnValidator;
72 int m_NumUrls;
73 char m_aaUrls[MAX_URLS][256];
74 };
75 class CJob : public IJob
76 {
77 CChooseMaster *m_pParent;
78 CLock m_Lock;
79 std::shared_ptr<CData> m_pData;
80 std::shared_ptr<CHttpRequest> m_pHead;
81 std::shared_ptr<CHttpRequest> m_pGet;
82 void Run() override REQUIRES(!m_Lock);
83
84 public:
85 CJob(CChooseMaster *pParent, std::shared_ptr<CData> pData) :
86 m_pParent(pParent),
87 m_pData(std::move(pData))
88 {
89 Abortable(Abortable: true);
90 }
91 bool Abort() override REQUIRES(!m_Lock);
92 };
93
94 IEngine *m_pEngine;
95 IHttp *m_pHttp;
96 int m_PreviousBestIndex;
97 std::shared_ptr<CData> m_pData;
98 std::shared_ptr<CJob> m_pJob;
99};
100
101CChooseMaster::CChooseMaster(IEngine *pEngine, IHttp *pHttp, VALIDATOR pfnValidator, const char **ppUrls, int NumUrls, int PreviousBestIndex) :
102 m_pEngine(pEngine),
103 m_pHttp(pHttp),
104 m_PreviousBestIndex(PreviousBestIndex)
105{
106 dbg_assert(NumUrls >= 0, "no master URLs");
107 dbg_assert(NumUrls <= MAX_URLS, "too many master URLs");
108 dbg_assert(PreviousBestIndex >= -1, "previous best index negative and not -1");
109 dbg_assert(PreviousBestIndex < NumUrls, "previous best index too high");
110 m_pData = std::make_shared<CData>();
111 m_pData->m_pfnValidator = pfnValidator;
112 m_pData->m_NumUrls = NumUrls;
113 for(int i = 0; i < m_pData->m_NumUrls; i++)
114 {
115 str_copy(dst&: m_pData->m_aaUrls[i], src: ppUrls[i]);
116 }
117}
118
119CChooseMaster::~CChooseMaster()
120{
121 if(m_pJob)
122 {
123 m_pJob->Abort();
124 }
125}
126
127int CChooseMaster::GetBestIndex() const
128{
129 int BestIndex = m_pData->m_BestIndex.load();
130 if(BestIndex >= 0)
131 {
132 return BestIndex;
133 }
134 else
135 {
136 return m_PreviousBestIndex;
137 }
138}
139
140bool CChooseMaster::GetBestUrl(const char **ppBestUrl) const
141{
142 int Index = GetBestIndex();
143 if(Index < 0)
144 {
145 *ppBestUrl = nullptr;
146 return true;
147 }
148 *ppBestUrl = m_pData->m_aaUrls[Index];
149 return false;
150}
151
152void CChooseMaster::Reset()
153{
154 m_PreviousBestIndex = -1;
155 m_pData->m_BestIndex.store(i: -1);
156}
157
158void CChooseMaster::Refresh()
159{
160 if(m_pJob == nullptr || m_pJob->State() == IJob::STATE_DONE)
161 {
162 m_pJob = std::make_shared<CJob>(args: this, args&: m_pData);
163 m_pEngine->AddJob(pJob: m_pJob);
164 }
165}
166
167bool CChooseMaster::CJob::Abort()
168{
169 if(!IJob::Abort())
170 {
171 return false;
172 }
173
174 CLockScope ls(m_Lock);
175 if(m_pHead != nullptr)
176 {
177 m_pHead->Abort();
178 }
179
180 if(m_pGet != nullptr)
181 {
182 m_pGet->Abort();
183 }
184
185 return true;
186}
187
188void CChooseMaster::CJob::Run()
189{
190 // Check masters in a random order.
191 int aRandomized[MAX_URLS] = {0};
192 for(int i = 0; i < m_pData->m_NumUrls; i++)
193 {
194 aRandomized[i] = i;
195 }
196 // https://en.wikipedia.org/w/index.php?title=Fisher%E2%80%93Yates_shuffle&oldid=1002922479#The_modern_algorithm
197 // The equivalent version.
198 for(int i = 0; i <= m_pData->m_NumUrls - 2; i++)
199 {
200 int j = i + secure_rand_below(below: m_pData->m_NumUrls - i);
201 std::swap(a&: aRandomized[i], b&: aRandomized[j]);
202 }
203 // Do a HEAD request to ensure that a connection is established and
204 // then do a GET request to check how fast we can get the server list.
205 //
206 // 10 seconds connection timeout, lower than 8KB/s for 10 seconds to
207 // fail.
208 CTimeout Timeout{.ConnectTimeoutMs: 10000, .TimeoutMs: 0, .LowSpeedLimit: 8000, .LowSpeedTime: 10};
209 int aTimeMs[MAX_URLS];
210 int aAgeS[MAX_URLS];
211 for(int i = 0; i < m_pData->m_NumUrls; i++)
212 {
213 aTimeMs[i] = -1;
214 aAgeS[i] = SanitizeAge(Age: {});
215 const char *pUrl = m_pData->m_aaUrls[aRandomized[i]];
216 std::shared_ptr<CHttpRequest> pHead = HttpHead(pUrl);
217 pHead->Timeout(Timeout);
218 pHead->LogProgress(LogProgress: HTTPLOG::FAILURE);
219 {
220 CLockScope ls(m_Lock);
221 m_pHead = pHead;
222 }
223
224 m_pParent->m_pHttp->Run(pRequest: pHead);
225 pHead->Wait();
226 if(pHead->State() == EHttpState::ABORTED || State() == IJob::STATE_ABORTED)
227 {
228 log_debug("serverbrowser_http", "master chooser aborted");
229 return;
230 }
231 if(pHead->State() != EHttpState::DONE)
232 {
233 continue;
234 }
235
236 auto StartTime = time_get_nanoseconds();
237 std::shared_ptr<CHttpRequest> pGet = HttpGet(pUrl);
238 pGet->Timeout(Timeout);
239 pGet->LogProgress(LogProgress: HTTPLOG::FAILURE);
240 {
241 CLockScope ls(m_Lock);
242 m_pGet = pGet;
243 }
244
245 m_pParent->m_pHttp->Run(pRequest: pGet);
246 pGet->Wait();
247
248 auto Time = std::chrono::duration_cast<std::chrono::milliseconds>(d: time_get_nanoseconds() - StartTime);
249 if(pGet->State() == EHttpState::ABORTED || State() == IJob::STATE_ABORTED)
250 {
251 log_debug("serverbrowser_http", "master chooser aborted");
252 return;
253 }
254 if(pGet->State() != EHttpState::DONE)
255 {
256 continue;
257 }
258 json_value *pJson = pGet->ResultJson();
259 if(!pJson)
260 {
261 continue;
262 }
263
264 bool ParseFailure = m_pData->m_pfnValidator(pJson);
265 json_value_free(pJson);
266 if(ParseFailure)
267 {
268 continue;
269 }
270 int AgeS = SanitizeAge(Age: pGet->ResultAgeSeconds());
271 log_info("serverbrowser_http", "found master, url='%s' time=%dms age=%ds", pUrl, (int)Time.count(), AgeS);
272
273 aTimeMs[i] = Time.count();
274 aAgeS[i] = AgeS;
275 }
276
277 // Determine index of the minimum time.
278 int BestIndex = -1;
279 int BestTime = 0;
280 int BestAge = 0;
281 for(int i = 0; i < m_pData->m_NumUrls; i++)
282 {
283 if(aTimeMs[i] < 0)
284 {
285 continue;
286 }
287 if(BestIndex == -1 || std::tuple(ClassifyAge(AgeSeconds: aAgeS[i]), aTimeMs[i]) < std::tuple(ClassifyAge(AgeSeconds: BestAge), BestTime))
288 {
289 BestTime = aTimeMs[i];
290 BestAge = aAgeS[i];
291 BestIndex = aRandomized[i];
292 }
293 }
294 if(BestIndex == -1)
295 {
296 log_error("serverbrowser_http", "WARNING: no usable masters found");
297 return;
298 }
299
300 log_info("serverbrowser_http", "determined best master, url='%s' time=%dms age=%ds", m_pData->m_aaUrls[BestIndex], BestTime, BestAge);
301 m_pData->m_BestIndex.store(i: BestIndex);
302}
303
304class CServerBrowserHttp : public IServerBrowserHttp
305{
306public:
307 CServerBrowserHttp(IEngine *pEngine, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex);
308 ~CServerBrowserHttp() override;
309 void Update() override;
310 bool IsRefreshing() override { return m_State != STATE_DONE; }
311 void Refresh() override;
312 bool GetBestUrl(const char **pBestUrl) const override { return m_pChooseMaster->GetBestUrl(ppBestUrl: pBestUrl); }
313
314 int NumServers() const override
315 {
316 return m_vServers.size();
317 }
318 const CServerInfo &Server(int Index) const override
319 {
320 return m_vServers[Index];
321 }
322 int NumLegacyServers() const override
323 {
324 return m_vLegacyServers.size();
325 }
326 const NETADDR &LegacyServer(int Index) const override
327 {
328 return m_vLegacyServers[Index];
329 }
330
331private:
332 enum
333 {
334 STATE_DONE,
335 STATE_WANTREFRESH,
336 STATE_REFRESHING,
337 STATE_NO_MASTER,
338 };
339
340 static bool Validate(json_value *pJson);
341 static bool Parse(json_value *pJson, std::vector<CServerInfo> *pvServers, std::vector<NETADDR> *pvLegacyServers);
342
343 IHttp *m_pHttp;
344
345 int m_State = STATE_DONE;
346 std::shared_ptr<CHttpRequest> m_pGetServers;
347 std::unique_ptr<CChooseMaster> m_pChooseMaster;
348
349 std::vector<CServerInfo> m_vServers;
350 std::vector<NETADDR> m_vLegacyServers;
351};
352
353CServerBrowserHttp::CServerBrowserHttp(IEngine *pEngine, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex) :
354 m_pHttp(pHttp),
355 m_pChooseMaster(new CChooseMaster(pEngine, pHttp, Validate, ppUrls, NumUrls, PreviousBestIndex))
356{
357 m_pChooseMaster->Refresh();
358}
359
360CServerBrowserHttp::~CServerBrowserHttp()
361{
362 if(m_pGetServers != nullptr)
363 {
364 m_pGetServers->Abort();
365 }
366}
367
368void CServerBrowserHttp::Update()
369{
370 if(m_State == STATE_WANTREFRESH)
371 {
372 const char *pBestUrl;
373 if(m_pChooseMaster->GetBestUrl(ppBestUrl: &pBestUrl))
374 {
375 if(!m_pChooseMaster->IsRefreshing())
376 {
377 log_error("serverbrowser_http", "no working serverlist URL found");
378 m_State = STATE_NO_MASTER;
379 }
380 return;
381 }
382 m_pGetServers = HttpGet(pUrl: pBestUrl);
383 // 10 seconds connection timeout, lower than 8KB/s for 10 seconds to fail.
384 m_pGetServers->Timeout(Timeout: CTimeout{.ConnectTimeoutMs: 10000, .TimeoutMs: 0, .LowSpeedLimit: 8000, .LowSpeedTime: 10});
385 m_pHttp->Run(pRequest: m_pGetServers);
386 m_State = STATE_REFRESHING;
387 }
388 else if(m_State == STATE_REFRESHING)
389 {
390 if(!m_pGetServers->Done())
391 {
392 return;
393 }
394 m_State = STATE_DONE;
395 std::shared_ptr<CHttpRequest> pGetServers = nullptr;
396 std::swap(a&: m_pGetServers, b&: pGetServers);
397
398 bool Success = true;
399 json_value *pJson = pGetServers->State() == EHttpState::DONE ? pGetServers->ResultJson() : nullptr;
400 Success = Success && pJson;
401 Success = Success && !Parse(pJson, pvServers: &m_vServers, pvLegacyServers: &m_vLegacyServers);
402 json_value_free(pJson);
403 if(!Success)
404 {
405 log_error("serverbrowser_http", "failed getting serverlist, trying to find best URL");
406 m_pChooseMaster->Reset();
407 m_pChooseMaster->Refresh();
408 }
409 else
410 {
411 // Try to find new master if the current one returns
412 // results that are 5 minutes old.
413 int Age = SanitizeAge(Age: pGetServers->ResultAgeSeconds());
414 if(Age > 300)
415 {
416 log_info("serverbrowser_http", "got stale serverlist, age=%ds, trying to find best URL", Age);
417 m_pChooseMaster->Refresh();
418 }
419 }
420 }
421}
422void CServerBrowserHttp::Refresh()
423{
424 if(m_State == STATE_WANTREFRESH || m_State == STATE_REFRESHING || m_State == STATE_NO_MASTER)
425 {
426 if(m_State == STATE_NO_MASTER)
427 m_State = STATE_WANTREFRESH;
428 m_pChooseMaster->Refresh();
429 }
430 if(m_State == STATE_DONE)
431 m_State = STATE_WANTREFRESH;
432 Update();
433}
434bool ServerbrowserParseUrl(NETADDR *pOut, const char *pUrl)
435{
436 return net_addr_from_url(addr: pOut, string: pUrl, host_buf: nullptr, host_buf_size: 0) != 0;
437}
438bool CServerBrowserHttp::Validate(json_value *pJson)
439{
440 std::vector<CServerInfo> vServers;
441 std::vector<NETADDR> vLegacyServers;
442 return Parse(pJson, pvServers: &vServers, pvLegacyServers: &vLegacyServers);
443}
444bool CServerBrowserHttp::Parse(json_value *pJson, std::vector<CServerInfo> *pvServers, std::vector<NETADDR> *pvLegacyServers)
445{
446 std::vector<CServerInfo> vServers;
447 std::vector<NETADDR> vLegacyServers;
448
449 const json_value &Json = *pJson;
450 const json_value &Servers = Json["servers"];
451 const json_value &LegacyServers = Json["servers_legacy"];
452 if(Servers.type != json_array || (LegacyServers.type != json_array && LegacyServers.type != json_none))
453 {
454 return true;
455 }
456 for(unsigned int i = 0; i < Servers.u.array.length; i++)
457 {
458 const json_value &Server = Servers[i];
459 const json_value &Addresses = Server["addresses"];
460 const json_value &Info = Server["info"];
461 const json_value &Location = Server["location"];
462 int ParsedLocation = CServerInfo::LOC_UNKNOWN;
463 CServerInfo2 ParsedInfo;
464 if(Addresses.type != json_array || (Location.type != json_string && Location.type != json_none))
465 {
466 return true;
467 }
468 if(Location.type == json_string)
469 {
470 if(CServerInfo::ParseLocation(pResult: &ParsedLocation, pString: Location))
471 {
472 return true;
473 }
474 }
475 if(CServerInfo2::FromJson(pOut: &ParsedInfo, pJson: &Info))
476 {
477 log_debug("serverbrowser_http", "skipped due to info, i=%d", i);
478 // Only skip the current server on parsing
479 // failure; the server info is "user input" by
480 // the game server and can be set to arbitrary
481 // values.
482 continue;
483 }
484 CServerInfo SetInfo = ParsedInfo;
485 SetInfo.m_Location = ParsedLocation;
486 SetInfo.m_NumAddresses = 0;
487 for(unsigned int a = 0; a < Addresses.u.array.length; a++)
488 {
489 const json_value &Address = Addresses[a];
490 if(Address.type != json_string)
491 {
492 return true;
493 }
494 NETADDR ParsedAddr;
495 if(ServerbrowserParseUrl(pOut: &ParsedAddr, pUrl: Addresses[a]))
496 {
497 // log_debug("serverbrowser_http", "unknown address, i=%d a=%d", i, a);
498 // Skip unknown addresses.
499 continue;
500 }
501 if(SetInfo.m_NumAddresses < (int)std::size(SetInfo.m_aAddresses))
502 {
503 SetInfo.m_aAddresses[SetInfo.m_NumAddresses] = ParsedAddr;
504 SetInfo.m_NumAddresses += 1;
505 }
506 }
507 if(SetInfo.m_NumAddresses > 0)
508 {
509 vServers.push_back(x: SetInfo);
510 }
511 }
512 if(LegacyServers.type == json_array)
513 {
514 for(unsigned int i = 0; i < LegacyServers.u.array.length; i++)
515 {
516 const json_value &Address = LegacyServers[i];
517 NETADDR ParsedAddr;
518 if(Address.type != json_string || net_addr_from_str(addr: &ParsedAddr, string: Address))
519 {
520 return true;
521 }
522 vLegacyServers.push_back(x: ParsedAddr);
523 }
524 }
525 *pvServers = vServers;
526 *pvLegacyServers = vLegacyServers;
527 return false;
528}
529
530static const char *DEFAULT_SERVERLIST_URLS[] = {
531 "https://master1.ddnet.org/ddnet/15/servers.json",
532 "https://master2.ddnet.org/ddnet/15/servers.json",
533 "https://master3.ddnet.org/ddnet/15/servers.json",
534 "https://master4.ddnet.org/ddnet/15/servers.json",
535};
536
537IServerBrowserHttp *CreateServerBrowserHttp(IEngine *pEngine, IStorage *pStorage, IHttp *pHttp, const char *pPreviousBestUrl)
538{
539 char aaUrls[CChooseMaster::MAX_URLS][256];
540 const char *apUrls[CChooseMaster::MAX_URLS] = {0};
541 const char **ppUrls = apUrls;
542 int NumUrls = 0;
543 CLineReader LineReader;
544 if(LineReader.OpenFile(File: pStorage->OpenFile(pFilename: "ddnet-serverlist-urls.cfg", Flags: IOFLAG_READ, Type: IStorage::TYPE_ALL)))
545 {
546 while(const char *pLine = LineReader.Get())
547 {
548 if(NumUrls == CChooseMaster::MAX_URLS)
549 {
550 break;
551 }
552 str_copy(dst&: aaUrls[NumUrls], src: pLine);
553 apUrls[NumUrls] = aaUrls[NumUrls];
554 NumUrls += 1;
555 }
556 }
557 if(NumUrls == 0)
558 {
559 ppUrls = DEFAULT_SERVERLIST_URLS;
560 NumUrls = std::size(DEFAULT_SERVERLIST_URLS);
561 }
562 int PreviousBestIndex = -1;
563 for(int i = 0; i < NumUrls; i++)
564 {
565 if(str_comp(a: ppUrls[i], b: pPreviousBestUrl) == 0)
566 {
567 PreviousBestIndex = i;
568 break;
569 }
570 }
571 return new CServerBrowserHttp(pEngine, pHttp, ppUrls, NumUrls, PreviousBestIndex);
572}
573