1#ifndef ENGINE_SHARED_HTTP_H
2#define ENGINE_SHARED_HTTP_H
3
4#include <base/hash_ctxt.h>
5#include <base/system.h>
6
7#include <engine/http.h>
8#include <engine/shared/jobs.h>
9
10#include <algorithm>
11#include <atomic>
12#include <condition_variable>
13#include <deque>
14#include <mutex>
15#include <optional>
16#include <unordered_map>
17
18typedef struct _json_value json_value;
19class IStorage;
20
21enum class EHttpState
22{
23 ERROR = -1,
24 QUEUED,
25 RUNNING,
26 DONE,
27 ABORTED,
28};
29
30enum class HTTPLOG
31{
32 NONE,
33 FAILURE,
34 ALL,
35};
36
37enum class IPRESOLVE
38{
39 WHATEVER,
40 V4,
41 V6,
42};
43
44class CTimeout
45{
46public:
47 long m_ConnectTimeoutMs;
48 long m_TimeoutMs;
49 long m_LowSpeedLimit;
50 long m_LowSpeedTime;
51};
52
53class CHttpRequest : public IHttpRequest
54{
55 friend class CHttp;
56
57 enum class REQUEST
58 {
59 GET = 0,
60 HEAD,
61 POST,
62 POST_JSON,
63 };
64
65 static constexpr const char *GetRequestType(REQUEST Type)
66 {
67 switch(Type)
68 {
69 case REQUEST::GET:
70 return "GET";
71 case REQUEST::HEAD:
72 return "HEAD";
73 case REQUEST::POST:
74 case REQUEST::POST_JSON:
75 return "POST";
76 }
77
78 dbg_assert_failed("unreachable");
79 }
80
81 char m_aUrl[256] = {0};
82
83 void *m_pHeaders = nullptr;
84 unsigned char *m_pBody = nullptr;
85 size_t m_BodyLength = 0;
86
87 bool m_ValidateBeforeOverwrite = false;
88 bool m_SkipByFileTime = true;
89
90 CTimeout m_Timeout = CTimeout{.m_ConnectTimeoutMs: 0, .m_TimeoutMs: 0, .m_LowSpeedLimit: 0, .m_LowSpeedTime: 0};
91 int64_t m_MaxResponseSize = -1;
92 int64_t m_IfModifiedSince = -1;
93 REQUEST m_Type = REQUEST::GET;
94
95 SHA256_DIGEST m_ActualSha256 = SHA256_ZEROED;
96 SHA256_CTX m_ActualSha256Ctx;
97 SHA256_DIGEST m_ExpectedSha256 = SHA256_ZEROED;
98
99 bool m_WriteToMemory = true;
100 bool m_WriteToFile = false;
101
102 uint64_t m_ResponseLength = 0;
103
104 // If `m_WriteToMemory` is true.
105 size_t m_BufferSize = 0;
106 unsigned char *m_pBuffer = nullptr;
107
108 // If `m_WriteToFile` is true.
109 IOHANDLE m_File = nullptr;
110 int m_StorageType = 0xdeadbeef;
111 char m_aDestAbsoluteTmp[IO_MAX_PATH_LENGTH] = {0};
112 char m_aDestAbsolute[IO_MAX_PATH_LENGTH] = {0};
113 char m_aDest[IO_MAX_PATH_LENGTH] = {0};
114
115 std::atomic<double> m_Size{0.0};
116 std::atomic<double> m_Current{0.0};
117 std::atomic<int> m_Progress{0};
118 HTTPLOG m_LogProgress = HTTPLOG::ALL;
119 IPRESOLVE m_IpResolve = IPRESOLVE::WHATEVER;
120
121 bool m_FailOnErrorStatus = true;
122
123 char m_aErr[256]; // 256 == CURL_ERROR_SIZE
124 std::atomic<EHttpState> m_State{EHttpState::QUEUED};
125 std::atomic<bool> m_Abort{false};
126 std::mutex m_WaitMutex;
127 std::condition_variable m_WaitCondition;
128
129 int m_StatusCode = 0;
130 bool m_HeadersEnded = false;
131 std::optional<int64_t> m_ResultDate = std::nullopt;
132 std::optional<int64_t> m_ResultLastModified = std::nullopt;
133
134 bool ShouldSkipRequest();
135 // Abort the request with an error if `BeforeInit()` returns false.
136 bool BeforeInit();
137 bool ConfigureHandle(void *pHandle); // void * == CURL *
138 // `pHandle` can be nullptr if no handle was ever created for this request.
139 void OnCompletionInternal(void *pHandle, unsigned int Result); // void * == CURL *, unsigned int == CURLcode
140
141 // Abort the request if `OnHeader()` returns something other than
142 // `DataSize`. `pHeader` is NOT null-terminated.
143 size_t OnHeader(char *pHeader, size_t HeaderSize);
144 // Abort the request if `OnData()` returns something other than
145 // `DataSize`.
146 size_t OnData(char *pData, size_t DataSize);
147
148 static int ProgressCallback(void *pUser, double DlTotal, double DlCurr, double UlTotal, double UlCurr);
149 static size_t HeaderCallback(char *pData, size_t Size, size_t Number, void *pUser);
150 static size_t WriteCallback(char *pData, size_t Size, size_t Number, void *pUser);
151
152protected:
153 // These run on the curl thread now, DO NOT STALL THE THREAD
154 virtual void OnProgress() {}
155 virtual void OnCompletion(EHttpState State) {}
156
157public:
158 CHttpRequest(const char *pUrl);
159 virtual ~CHttpRequest();
160
161 void Timeout(CTimeout Timeout) { m_Timeout = Timeout; }
162 // Skip the download if the local file is newer or as new as the remote file.
163 void SkipByFileTime(bool SkipByFileTime) { m_SkipByFileTime = SkipByFileTime; }
164 void MaxResponseSize(int64_t MaxResponseSize) { m_MaxResponseSize = MaxResponseSize; }
165 void LogProgress(HTTPLOG LogProgress) { m_LogProgress = LogProgress; }
166 void IpResolve(IPRESOLVE IpResolve) { m_IpResolve = IpResolve; }
167 void FailOnErrorStatus(bool FailOnErrorStatus) { m_FailOnErrorStatus = FailOnErrorStatus; }
168 // Download to memory only. Get the result via `Result*`.
169 void WriteToMemory()
170 {
171 m_WriteToMemory = true;
172 m_WriteToFile = false;
173 }
174 // Download to filesystem and memory.
175 void WriteToFileAndMemory(IStorage *pStorage, const char *pDest, int StorageType);
176 // Download to the filesystem only.
177 void WriteToFile(IStorage *pStorage, const char *pDest, int StorageType);
178 // Don't place the file in the specified location until
179 // `OnValidation(true)` has been called.
180 void ValidateBeforeOverwrite(bool ValidateBeforeOverwrite) { m_ValidateBeforeOverwrite = ValidateBeforeOverwrite; }
181 void ExpectSha256(const SHA256_DIGEST &Sha256) { m_ExpectedSha256 = Sha256; }
182 void Head() { m_Type = REQUEST::HEAD; }
183 void Post(const unsigned char *pData, size_t DataLength)
184 {
185 m_Type = REQUEST::POST;
186 m_BodyLength = DataLength;
187 m_pBody = (unsigned char *)malloc(size: std::max(a: (size_t)1, b: DataLength));
188 mem_copy(dest: m_pBody, source: pData, size: DataLength);
189 }
190 void PostJson(const char *pJson)
191 {
192 m_Type = REQUEST::POST_JSON;
193 m_BodyLength = str_length(str: pJson);
194 m_pBody = (unsigned char *)malloc(size: m_BodyLength);
195 mem_copy(dest: m_pBody, source: pJson, size: m_BodyLength);
196 }
197 void Header(const char *pNameColonValue);
198 void HeaderString(const char *pName, const char *pValue)
199 {
200 char aHeader[256];
201 str_format(buffer: aHeader, buffer_size: sizeof(aHeader), format: "%s: %s", pName, pValue);
202 Header(pNameColonValue: aHeader);
203 }
204 void HeaderInt(const char *pName, int Value)
205 {
206 char aHeader[256];
207 str_format(buffer: aHeader, buffer_size: sizeof(aHeader), format: "%s: %d", pName, Value);
208 Header(pNameColonValue: aHeader);
209 }
210
211 const char *Dest()
212 {
213 if(m_WriteToFile)
214 {
215 return m_aDest;
216 }
217 else
218 {
219 return nullptr;
220 }
221 }
222
223 double Current() const { return m_Current.load(m: std::memory_order_relaxed); }
224 double Size() const { return m_Size.load(m: std::memory_order_relaxed); }
225 int Progress() const { return m_Progress.load(m: std::memory_order_relaxed); }
226 EHttpState State() const { return m_State; }
227 bool Done() const
228 {
229 EHttpState State = m_State;
230 return State != EHttpState::QUEUED && State != EHttpState::RUNNING;
231 }
232 void Abort() { m_Abort = true; }
233 // If `ValidateBeforeOverwrite` is set, this needs to be called after
234 // validating that the downloaded file has the correct format.
235 //
236 // If called with `true`, it'll place the downloaded file at the final
237 // destination, if called with `false`, it'll instead delete the
238 // temporary downloaded file.
239 void OnValidation(bool Success);
240
241 void Wait();
242
243 void Result(unsigned char **ppResult, size_t *pResultLength) const;
244 json_value *ResultJson() const;
245 const SHA256_DIGEST &ResultSha256() const;
246
247 int StatusCode() const;
248 std::optional<int64_t> ResultAgeSeconds() const;
249 std::optional<int64_t> ResultLastModified() const;
250};
251
252inline std::unique_ptr<CHttpRequest> HttpHead(const char *pUrl)
253{
254 auto pResult = std::make_unique<CHttpRequest>(args&: pUrl);
255 pResult->Head();
256 return pResult;
257}
258
259inline std::unique_ptr<CHttpRequest> HttpGet(const char *pUrl)
260{
261 return std::make_unique<CHttpRequest>(args&: pUrl);
262}
263
264inline std::unique_ptr<CHttpRequest> HttpGetFile(const char *pUrl, IStorage *pStorage, const char *pOutputFile, int StorageType)
265{
266 std::unique_ptr<CHttpRequest> pResult = HttpGet(pUrl);
267 pResult->WriteToFile(pStorage, pDest: pOutputFile, StorageType);
268 pResult->Timeout(Timeout: CTimeout{.m_ConnectTimeoutMs: 4000, .m_TimeoutMs: 0, .m_LowSpeedLimit: 500, .m_LowSpeedTime: 5});
269 return pResult;
270}
271
272inline std::unique_ptr<CHttpRequest> HttpGetBoth(const char *pUrl, IStorage *pStorage, const char *pOutputFile, int StorageType)
273{
274 std::unique_ptr<CHttpRequest> pResult = HttpGet(pUrl);
275 pResult->WriteToFileAndMemory(pStorage, pDest: pOutputFile, StorageType);
276 pResult->Timeout(Timeout: CTimeout{.m_ConnectTimeoutMs: 4000, .m_TimeoutMs: 0, .m_LowSpeedLimit: 500, .m_LowSpeedTime: 5});
277 return pResult;
278}
279
280inline std::unique_ptr<CHttpRequest> HttpPost(const char *pUrl, const unsigned char *pData, size_t DataLength)
281{
282 auto pResult = std::make_unique<CHttpRequest>(args&: pUrl);
283 pResult->Post(pData, DataLength);
284 pResult->Timeout(Timeout: CTimeout{.m_ConnectTimeoutMs: 4000, .m_TimeoutMs: 15000, .m_LowSpeedLimit: 500, .m_LowSpeedTime: 5});
285 return pResult;
286}
287
288inline std::unique_ptr<CHttpRequest> HttpPostJson(const char *pUrl, const char *pJson)
289{
290 auto pResult = std::make_unique<CHttpRequest>(args&: pUrl);
291 pResult->PostJson(pJson);
292 pResult->Timeout(Timeout: CTimeout{.m_ConnectTimeoutMs: 4000, .m_TimeoutMs: 15000, .m_LowSpeedLimit: 500, .m_LowSpeedTime: 5});
293 return pResult;
294}
295
296void EscapeUrl(char *pBuf, int Size, const char *pStr);
297
298template<int N>
299void EscapeUrl(char (&aBuf)[N], const char *pStr)
300{
301 EscapeUrl(aBuf, N, pStr);
302}
303
304bool HttpHasIpresolveBug();
305
306// In an ideal world this would be a kernel interface
307class CHttp : public IHttp
308{
309 enum EState
310 {
311 UNINITIALIZED,
312 RUNNING,
313 ERROR,
314 };
315
316 void *m_pThread = nullptr;
317
318 std::mutex m_Lock;
319 std::condition_variable m_Cv;
320 std::atomic<EState> m_State = UNINITIALIZED;
321 std::deque<std::shared_ptr<CHttpRequest>> m_PendingRequests;
322 std::unordered_map<void *, std::shared_ptr<CHttpRequest>> m_RunningRequests; // void * == CURL *
323 std::chrono::milliseconds m_ShutdownDelay{};
324 std::optional<std::chrono::time_point<std::chrono::steady_clock>> m_ShutdownTime;
325 std::atomic<bool> m_Shutdown = false;
326
327 // Only to be used with curl_multi_wakeup
328 void *m_pMultiH = nullptr; // void * == CURLM *
329
330 static void ThreadMain(void *pUser);
331 void RunLoop();
332
333public:
334 // Startup
335 bool Init(std::chrono::milliseconds ShutdownDelay);
336
337 // User
338 void Run(std::shared_ptr<IHttpRequest> pRequest) override;
339 void Shutdown() override;
340 ~CHttp() override;
341};
342
343#endif // ENGINE_SHARED_HTTP_H
344