1#ifndef ENGINE_SHARED_HTTP_H
2#define ENGINE_SHARED_HTTP_H
3
4#include <base/dbg.h>
5#include <base/hash_ctxt.h>
6#include <base/mem.h>
7#include <base/str.h>
8#include <base/types.h>
9
10#include <engine/http.h>
11#include <engine/shared/jobs.h>
12
13#include <algorithm>
14#include <atomic>
15#include <condition_variable>
16#include <cstdlib>
17#include <deque>
18#include <mutex>
19#include <optional>
20#include <unordered_map>
21
22typedef struct _json_value json_value;
23class IStorage;
24
25enum class EHttpState
26{
27 ERROR = -1,
28 QUEUED,
29 RUNNING,
30 DONE,
31 ABORTED,
32};
33
34enum class HTTPLOG
35{
36 NONE,
37 FAILURE,
38 ALL,
39};
40
41enum class IPRESOLVE
42{
43 WHATEVER,
44 V4,
45 V6,
46};
47
48class CTimeout
49{
50public:
51 long m_ConnectTimeoutMs;
52 long m_TimeoutMs;
53 long m_LowSpeedLimit;
54 long m_LowSpeedTime;
55};
56
57class CHttpRequest : public IHttpRequest
58{
59 friend class CHttp;
60
61 enum class REQUEST
62 {
63 GET = 0,
64 HEAD,
65 POST,
66 POST_JSON,
67 };
68
69 static constexpr const char *GetRequestType(REQUEST Type)
70 {
71 switch(Type)
72 {
73 case REQUEST::GET:
74 return "GET";
75 case REQUEST::HEAD:
76 return "HEAD";
77 case REQUEST::POST:
78 case REQUEST::POST_JSON:
79 return "POST";
80 }
81
82 dbg_assert_failed("unreachable");
83 }
84
85 char m_aUrl[256] = {0};
86
87 void *m_pHeaders = nullptr;
88 unsigned char *m_pBody = nullptr;
89 size_t m_BodyLength = 0;
90
91 bool m_ValidateBeforeOverwrite = false;
92 bool m_SkipByFileTime = true;
93
94 CTimeout m_Timeout = CTimeout{.m_ConnectTimeoutMs: 0, .m_TimeoutMs: 0, .m_LowSpeedLimit: 0, .m_LowSpeedTime: 0};
95 int64_t m_MaxResponseSize = -1;
96 int64_t m_IfModifiedSince = -1;
97 REQUEST m_Type = REQUEST::GET;
98
99 std::optional<SHA256_DIGEST> m_ActualSha256;
100 SHA256_CTX m_ActualSha256Ctx;
101 std::optional<SHA256_DIGEST> m_ExpectedSha256;
102
103 bool m_WriteToMemory = true;
104 bool m_WriteToFile = false;
105
106 uint64_t m_ResponseLength = 0;
107
108 // If `m_WriteToMemory` is true.
109 size_t m_BufferSize = 0;
110 unsigned char *m_pBuffer = nullptr;
111
112 // If `m_WriteToFile` is true.
113 IOHANDLE m_File = nullptr;
114 int m_StorageType = 0xdeadbeef;
115 char m_aDestAbsoluteTmp[IO_MAX_PATH_LENGTH] = {0};
116 char m_aDestAbsolute[IO_MAX_PATH_LENGTH] = {0};
117 char m_aDest[IO_MAX_PATH_LENGTH] = {0};
118
119 std::atomic<double> m_Size{0.0};
120 std::atomic<double> m_Current{0.0};
121 std::atomic<int> m_Progress{0};
122 HTTPLOG m_LogProgress = HTTPLOG::ALL;
123 IPRESOLVE m_IpResolve = IPRESOLVE::WHATEVER;
124
125 bool m_FailOnErrorStatus = true;
126
127 char m_aErr[256]; // 256 == CURL_ERROR_SIZE
128 std::atomic<EHttpState> m_State{EHttpState::QUEUED};
129 std::atomic<bool> m_Abort{false};
130 std::mutex m_WaitMutex;
131 std::condition_variable m_WaitCondition;
132
133 int m_StatusCode = 0;
134 bool m_HeadersEnded = false;
135 std::optional<int64_t> m_ResultDate = std::nullopt;
136 std::optional<int64_t> m_ResultLastModified = std::nullopt;
137
138 bool ShouldSkipRequest();
139 // Abort the request with an error if `BeforeInit()` returns false.
140 bool BeforeInit();
141 bool ConfigureHandle(void *pHandle); // void * == CURL *
142 // `pHandle` can be nullptr if no handle was ever created for this request.
143 void OnCompletionInternal(void *pHandle, unsigned int Result); // void * == CURL *, unsigned int == CURLcode
144
145 // Abort the request if `OnHeader()` returns something other than
146 // `DataSize`. `pHeader` is NOT null-terminated.
147 size_t OnHeader(char *pHeader, size_t HeaderSize);
148 // Abort the request if `OnData()` returns something other than
149 // `DataSize`.
150 size_t OnData(char *pData, size_t DataSize);
151
152 static int ProgressCallback(void *pUser, double DlTotal, double DlCurr, double UlTotal, double UlCurr);
153 static size_t HeaderCallback(char *pData, size_t Size, size_t Number, void *pUser);
154 static size_t WriteCallback(char *pData, size_t Size, size_t Number, void *pUser);
155
156protected:
157 // These run on the curl thread now, DO NOT STALL THE THREAD
158 virtual void OnProgress() {}
159 virtual void OnCompletion(EHttpState State) {}
160
161public:
162 CHttpRequest(const char *pUrl);
163 virtual ~CHttpRequest();
164
165 void Timeout(CTimeout Timeout) { m_Timeout = Timeout; }
166 // Skip the download if the local file is newer or as new as the remote file.
167 void SkipByFileTime(bool SkipByFileTime) { m_SkipByFileTime = SkipByFileTime; }
168 void MaxResponseSize(int64_t MaxResponseSize) { m_MaxResponseSize = MaxResponseSize; }
169 void LogProgress(HTTPLOG LogProgress) { m_LogProgress = LogProgress; }
170 void IpResolve(IPRESOLVE IpResolve) { m_IpResolve = IpResolve; }
171 void FailOnErrorStatus(bool FailOnErrorStatus) { m_FailOnErrorStatus = FailOnErrorStatus; }
172 // Download to memory only. Get the result via `Result*`.
173 void WriteToMemory()
174 {
175 m_WriteToMemory = true;
176 m_WriteToFile = false;
177 }
178 // Download to filesystem and memory.
179 void WriteToFileAndMemory(IStorage *pStorage, const char *pDest, int StorageType);
180 // Download to the filesystem only.
181 void WriteToFile(IStorage *pStorage, const char *pDest, int StorageType);
182 // Don't place the file in the specified location until
183 // `OnValidation(true)` has been called.
184 void ValidateBeforeOverwrite(bool ValidateBeforeOverwrite) { m_ValidateBeforeOverwrite = ValidateBeforeOverwrite; }
185 void ExpectSha256(const SHA256_DIGEST &Sha256) { m_ExpectedSha256 = Sha256; }
186 void Head() { m_Type = REQUEST::HEAD; }
187 void Post(const unsigned char *pData, size_t DataLength)
188 {
189 m_Type = REQUEST::POST;
190 m_BodyLength = DataLength;
191 m_pBody = (unsigned char *)malloc(size: std::max(a: (size_t)1, b: DataLength));
192 mem_copy(dest: m_pBody, source: pData, size: DataLength);
193 }
194 void PostJson(const char *pJson)
195 {
196 m_Type = REQUEST::POST_JSON;
197 m_BodyLength = str_length(str: pJson);
198 m_pBody = (unsigned char *)malloc(size: m_BodyLength);
199 mem_copy(dest: m_pBody, source: pJson, size: m_BodyLength);
200 }
201 void Header(const char *pNameColonValue);
202 void HeaderString(const char *pName, const char *pValue)
203 {
204 char aHeader[256];
205 str_format(buffer: aHeader, buffer_size: sizeof(aHeader), format: "%s: %s", pName, pValue);
206 Header(pNameColonValue: aHeader);
207 }
208 void HeaderInt(const char *pName, int Value)
209 {
210 char aHeader[256];
211 str_format(buffer: aHeader, buffer_size: sizeof(aHeader), format: "%s: %d", pName, Value);
212 Header(pNameColonValue: aHeader);
213 }
214
215 const char *Dest()
216 {
217 if(m_WriteToFile)
218 {
219 return m_aDest;
220 }
221 else
222 {
223 return nullptr;
224 }
225 }
226
227 double Current() const { return m_Current.load(m: std::memory_order_relaxed); }
228 double Size() const { return m_Size.load(m: std::memory_order_relaxed); }
229 int Progress() const { return m_Progress.load(m: std::memory_order_relaxed); }
230 EHttpState State() const { return m_State; }
231 bool Done() const
232 {
233 EHttpState State = m_State;
234 return State != EHttpState::QUEUED && State != EHttpState::RUNNING;
235 }
236 void Abort() { m_Abort = true; }
237 // If `ValidateBeforeOverwrite` is set, this needs to be called after
238 // validating that the downloaded file has the correct format.
239 //
240 // If called with `true`, it'll place the downloaded file at the final
241 // destination, if called with `false`, it'll instead delete the
242 // temporary downloaded file.
243 void OnValidation(bool Success);
244
245 void Wait();
246
247 void Result(unsigned char **ppResult, size_t *pResultLength) const;
248 json_value *ResultJson() const;
249 const SHA256_DIGEST &ResultSha256() const;
250
251 int StatusCode() const;
252 std::optional<int64_t> ResultAgeSeconds() const;
253 std::optional<int64_t> ResultLastModified() const;
254};
255
256inline std::unique_ptr<CHttpRequest> HttpHead(const char *pUrl)
257{
258 auto pResult = std::make_unique<CHttpRequest>(args&: pUrl);
259 pResult->Head();
260 return pResult;
261}
262
263inline std::unique_ptr<CHttpRequest> HttpGet(const char *pUrl)
264{
265 return std::make_unique<CHttpRequest>(args&: pUrl);
266}
267
268inline std::unique_ptr<CHttpRequest> HttpGetFile(const char *pUrl, IStorage *pStorage, const char *pOutputFile, int StorageType)
269{
270 std::unique_ptr<CHttpRequest> pResult = HttpGet(pUrl);
271 pResult->WriteToFile(pStorage, pDest: pOutputFile, StorageType);
272 pResult->Timeout(Timeout: CTimeout{.m_ConnectTimeoutMs: 4000, .m_TimeoutMs: 0, .m_LowSpeedLimit: 500, .m_LowSpeedTime: 5});
273 return pResult;
274}
275
276inline std::unique_ptr<CHttpRequest> HttpGetBoth(const char *pUrl, IStorage *pStorage, const char *pOutputFile, int StorageType)
277{
278 std::unique_ptr<CHttpRequest> pResult = HttpGet(pUrl);
279 pResult->WriteToFileAndMemory(pStorage, pDest: pOutputFile, StorageType);
280 pResult->Timeout(Timeout: CTimeout{.m_ConnectTimeoutMs: 4000, .m_TimeoutMs: 0, .m_LowSpeedLimit: 500, .m_LowSpeedTime: 5});
281 return pResult;
282}
283
284inline std::unique_ptr<CHttpRequest> HttpPost(const char *pUrl, const unsigned char *pData, size_t DataLength)
285{
286 auto pResult = std::make_unique<CHttpRequest>(args&: pUrl);
287 pResult->Post(pData, DataLength);
288 pResult->Timeout(Timeout: CTimeout{.m_ConnectTimeoutMs: 4000, .m_TimeoutMs: 15000, .m_LowSpeedLimit: 500, .m_LowSpeedTime: 5});
289 return pResult;
290}
291
292inline std::unique_ptr<CHttpRequest> HttpPostJson(const char *pUrl, const char *pJson)
293{
294 auto pResult = std::make_unique<CHttpRequest>(args&: pUrl);
295 pResult->PostJson(pJson);
296 pResult->Timeout(Timeout: CTimeout{.m_ConnectTimeoutMs: 4000, .m_TimeoutMs: 15000, .m_LowSpeedLimit: 500, .m_LowSpeedTime: 5});
297 return pResult;
298}
299
300void EscapeUrl(char *pBuf, int Size, const char *pStr);
301
302template<int N>
303void EscapeUrl(char (&aBuf)[N], const char *pStr)
304{
305 EscapeUrl(aBuf, N, pStr);
306}
307
308bool HttpHasIpresolveBug();
309
310// In an ideal world this would be a kernel interface
311class CHttp : public IHttp
312{
313 enum EState
314 {
315 UNINITIALIZED,
316 RUNNING,
317 ERROR,
318 };
319
320 void *m_pThread = nullptr;
321
322 std::mutex m_Lock;
323 std::condition_variable m_Cv;
324 std::atomic<EState> m_State = UNINITIALIZED;
325 std::deque<std::shared_ptr<CHttpRequest>> m_PendingRequests;
326 std::unordered_map<void *, std::shared_ptr<CHttpRequest>> m_RunningRequests; // void * == CURL *
327 std::chrono::milliseconds m_ShutdownDelay{};
328 std::optional<std::chrono::time_point<std::chrono::steady_clock>> m_ShutdownTime;
329 std::atomic<bool> m_Shutdown = false;
330
331 // Only to be used with curl_multi_wakeup
332 void *m_pMultiH = nullptr; // void * == CURLM *
333
334 static void ThreadMain(void *pUser);
335 void RunLoop();
336
337public:
338 // Startup
339 bool Init(std::chrono::milliseconds ShutdownDelay);
340
341 // User
342 void Run(std::shared_ptr<IHttpRequest> pRequest) override;
343 void Shutdown() override;
344 ~CHttp() override;
345};
346
347#endif // ENGINE_SHARED_HTTP_H
348