OpenTTD Source 20261112-master-gbcbec5e63f
http_curl.cpp
Go to the documentation of this file.
1/*
2 * This file is part of OpenTTD.
3 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <https://www.gnu.org/licenses/old-licenses/gpl-2.0>.
6 */
7
9
10#include "../../stdafx.h"
11#include "../../debug.h"
12#include "../../fileio_func.h"
13#include "../../thread.h"
14
15#include "http.h"
16#include "http_shared.h"
17#include "network_game_info.h"
18
19#include <atomic>
20#include <condition_variable>
21#include <curl/curl.h>
22#include <mutex>
23#include <queue>
24
25#include "../../safeguards.h"
26
27#if defined(UNIX)
29static constexpr std::initializer_list<std::string_view> _certificate_files = {
30 "/etc/ssl/certs/ca-certificates.crt"sv, // Debian/Ubuntu/Gentoo etc.
31 "/etc/pki/tls/certs/ca-bundle.crt"sv, // Fedora/RHEL 6
32 "/etc/ssl/ca-bundle.pem"sv, // OpenSUSE
33 "/etc/pki/tls/cacert.pem"sv, // OpenELEC
34 "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"sv, // CentOS/RHEL 7
35 "/etc/ssl/cert.pem"sv, // Alpine Linux
36};
38static constexpr std::initializer_list<std::string_view> _certificate_directories = {
39 "/etc/ssl/certs"sv, // SLES10/SLES11, https://golang.org/issue/12139
40 "/etc/pki/tls/certs"sv, // Fedora/RHEL
41 "/system/etc/security/cacerts"sv, // Android
42};
43#endif /* UNIX */
44
45static std::vector<HTTPThreadSafeCallback *> _http_callbacks;
46static std::vector<HTTPThreadSafeCallback *> _new_http_callbacks;
47static std::mutex _http_callback_mutex;
48static std::mutex _new_http_callback_mutex;
49
52public:
60 NetworkHTTPRequest(std::string_view uri, HTTPCallback *callback, std::string &&data) :
61 uri(uri),
63 data(std::move(data))
64 {
65 std::lock_guard<std::mutex> lock(_new_http_callback_mutex);
66 _new_http_callbacks.push_back(&this->callback);
67 }
68
71 {
72 std::lock_guard<std::mutex> lock(_http_callback_mutex);
73 _http_callbacks.erase(std::remove(_http_callbacks.begin(), _http_callbacks.end(), &this->callback), _http_callbacks.end());
74 }
75
76 const std::string uri;
78 const std::string data;
79};
80
81static std::thread _http_thread;
82static std::atomic<bool> _http_thread_exit = false;
83static std::queue<std::unique_ptr<NetworkHTTPRequest>> _http_requests;
84static std::mutex _http_mutex;
85static std::condition_variable _http_cv;
86#if defined(UNIX)
87static std::string _http_ca_file = "";
88static std::string _http_ca_path = "";
89#endif /* UNIX */
90
91
92/* static */ void NetworkHTTPSocketHandler::Connect(std::string_view uri, HTTPCallback *callback, std::string &&data)
93{
94#if defined(UNIX)
95 if (_http_ca_file.empty() && _http_ca_path.empty()) {
96 callback->OnFailure();
97 return;
98 }
99#endif /* UNIX */
100
101 std::lock_guard<std::mutex> lock(_http_mutex);
102 _http_requests.push(std::make_unique<NetworkHTTPRequest>(uri, callback, std::move(data)));
103 _http_cv.notify_one();
104}
105
107{
108 std::lock_guard<std::mutex> lock(_http_callback_mutex);
109
110 {
111 std::lock_guard<std::mutex> lock_new(_new_http_callback_mutex);
112 if (!_new_http_callbacks.empty()) {
113 /* We delay adding new callbacks, as HandleQueue() below might add a new callback. */
115 _new_http_callbacks.clear();
116 }
117 }
118
119 for (auto &callback : _http_callbacks) {
120 callback->HandleQueue();
121 }
122}
123
130void CurlSetOption(CURL *curl, auto option, auto value)
131{
132 CURLcode res = curl_easy_setopt(curl, option, value);
133 if (res != CURLE_OK) {
134 Debug(Facility::Net, Severity::Critical, "Could not execute curl_easy_setopt for {} [{}]", option, res);
135 }
136}
137
140{
141 CURL *curl = curl_easy_init();
142 assert(curl != nullptr);
143
144 for (;;) {
145 std::unique_lock<std::mutex> lock(_http_mutex);
146
147 /* Wait for a new request. */
148 while (_http_requests.empty() && !_http_thread_exit) {
149 _http_cv.wait(lock);
150 }
151 if (_http_thread_exit) break;
152
153 std::unique_ptr<NetworkHTTPRequest> request = std::move(_http_requests.front());
154 _http_requests.pop();
155
156 /* Release the lock, as we will take a while to process the request. */
157 lock.unlock();
158
159 /* Reset to default settings. */
160 curl_easy_reset(curl);
161 curl_slist *headers = nullptr;
162
164 CurlSetOption(curl, CURLOPT_VERBOSE, 1L);
165 }
166
167 /* Setup some default options. */
168 std::string user_agent = fmt::format("OpenTTD/{}", GetNetworkRevisionString());
169 CurlSetOption(curl, CURLOPT_USERAGENT, user_agent.c_str());
170 CurlSetOption(curl, CURLOPT_FOLLOWLOCATION, 1L);
171 CurlSetOption(curl, CURLOPT_MAXREDIRS, 5L);
172
173 /* Ensure we validate the certificate and hostname of the server. */
174#if defined(UNIX)
175 CurlSetOption(curl, CURLOPT_CAINFO, _http_ca_file.empty() ? nullptr : _http_ca_file.c_str());
176 CurlSetOption(curl, CURLOPT_CAPATH, _http_ca_path.empty() ? nullptr : _http_ca_path.c_str());
177#endif /* UNIX */
178 CurlSetOption(curl, CURLOPT_SSL_VERIFYHOST, 2);
179 CurlSetOption(curl, CURLOPT_SSL_VERIFYPEER, true);
180
181 /* Give the connection about 10 seconds to complete. */
182 CurlSetOption(curl, CURLOPT_CONNECTTIMEOUT, 10L);
183
184 /* Set a buffer of 100KiB, as the default of 16KiB seems a bit small. */
185 CurlSetOption(curl, CURLOPT_BUFFERSIZE, 100L * 1024L);
186
187 /* Fail our call if we don't receive a 2XX return value. */
188 CurlSetOption(curl, CURLOPT_FAILONERROR, 1L);
189
190 /* Prepare POST body and URI. */
191 if (!request->data.empty()) {
192 /* When the payload starts with a '{', it is a JSON payload. */
193 if (request->data.starts_with("{")) {
194 headers = curl_slist_append(headers, "Content-Type: application/json");
195 } else {
196 headers = curl_slist_append(headers, "Content-Type: application/x-www-form-urlencoded");
197 }
198
199 CurlSetOption(curl, CURLOPT_POST, 1L);
200 CurlSetOption(curl, CURLOPT_POSTFIELDS, request->data.c_str());
201 CurlSetOption(curl, CURLOPT_HTTPHEADER, headers);
202 }
203 CurlSetOption(curl, CURLOPT_URL, request->uri.c_str());
204
205 /* Setup our (C-style) callback function which we pipe back into the callback. */
206 CurlSetOption(curl, CURLOPT_WRITEFUNCTION, +[](char *ptr, size_t size, size_t nmemb, void *userdata) -> size_t {
207 Debug(Facility::Net, Severity::Debug2, "HTTP callback: {} bytes", size * nmemb);
208 HTTPThreadSafeCallback *callback = static_cast<HTTPThreadSafeCallback *>(userdata);
209
210 /* Copy the buffer out of CURL. OnReceiveData() will free it when done. */
211 std::unique_ptr<char[]> buffer = std::make_unique<char[]>(size * nmemb);
212 std::copy_n(ptr, size * nmemb, buffer.get());
213 callback->OnReceiveData(std::move(buffer), size * nmemb);
214
215 return size * nmemb;
216 });
217 CurlSetOption(curl, CURLOPT_WRITEDATA, &request->callback);
218
219 /* Create a callback from which we can cancel. Sadly, there is no other
220 * thread-safe way to do this. If the connection went idle, it can take
221 * up to a second before this callback is called. There is little we can
222 * do about this. */
223 CurlSetOption(curl, CURLOPT_NOPROGRESS, 0L);
224 CurlSetOption(curl, CURLOPT_XFERINFOFUNCTION, +[](void *userdata, curl_off_t /*dltotal*/, curl_off_t /*dlnow*/, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) -> int {
225 const HTTPThreadSafeCallback *callback = static_cast<HTTPThreadSafeCallback *>(userdata);
226 return (callback->cancelled || _http_thread_exit) ? 1 : 0;
227 });
228 CurlSetOption(curl, CURLOPT_XFERINFODATA, &request->callback);
229
230 /* Perform the request. */
231 CURLcode res = curl_easy_perform(curl);
232
233 curl_slist_free_all(headers);
234
235 if (res == CURLE_OK) {
236 Debug(Facility::Net, Severity::Error, "HTTP request succeeded");
237 request->callback.OnReceiveData(nullptr, 0);
238 } else {
239 long status_code = 0;
240 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status_code);
241
242 /* No need to be verbose about rate limiting. */
243 Debug(Facility::Net, (request->callback.cancelled || _http_thread_exit || status_code == HTTP_429_TOO_MANY_REQUESTS) ? Severity::Error : Severity::Critical, "HTTP request failed: status_code: {}, error: {}", status_code, curl_easy_strerror(res));
244 request->callback.OnFailure();
245 }
246
247 /* Wait till the callback tells us all data is dequeued, or _http_thread_exit has been set. */
248 request->callback.WaitTillEmptyOrCondition([]() -> bool {
249 return _http_thread_exit;
250 });
251 }
252
253 curl_easy_cleanup(curl);
254}
255
257{
258 curl_global_init(CURL_GLOBAL_DEFAULT);
259
260#if defined(UNIX)
261 /* Depending on the Linux distro, certificates can either be in
262 * a bundle or a folder, in a wide range of different locations.
263 * Try to find what location is used by this OS. */
264 for (auto &ca_file : _certificate_files) {
265 if (FileExists(ca_file)) {
266 _http_ca_file = ca_file;
267 break;
268 }
269 }
270 if (_http_ca_file.empty()) {
271 for (auto &ca_path : _certificate_directories) {
272 if (FileExists(ca_path)) {
273 _http_ca_path = ca_path;
274 break;
275 }
276 }
277 }
278 Debug(Facility::Net, Severity::Notice, "Using certificate file: {}", _http_ca_file.empty() ? "none" : _http_ca_file);
279 Debug(Facility::Net, Severity::Notice, "Using certificate path: {}", _http_ca_path.empty() ? "none" : _http_ca_path);
280
281 /* Tell the user why HTTPS will not be working. */
282 if (_http_ca_file.empty() && _http_ca_path.empty()) {
283 Debug(Facility::Net, Severity::Critical, "No certificate files or directories found, HTTPS will not work!");
284 }
285#endif /* UNIX */
286
287 _http_thread_exit = false;
288 StartNewThread(&_http_thread, "ottd:http", &HttpThread);
289}
290
292{
293 _http_thread_exit = true;
294
295 /* Ensure the callbacks are handled. This is mostly needed as we send
296 * a survey just before close, and that might be pending here. */
298
299 {
300 std::lock_guard<std::mutex> lock(_http_mutex);
301 _http_cv.notify_one();
302 }
303
304 if (_http_thread.joinable()) {
305 _http_thread.join();
306 }
307
308 curl_global_cleanup();
309}
Converts a HTTPCallback to a Thread-Safe variant.
Definition http_shared.h:20
void OnReceiveData(std::unique_ptr< char[]> data, size_t length)
Similar to HTTPCallback::OnReceiveData, but thread-safe.
Definition http_shared.h:53
std::atomic< bool > cancelled
Whether this callback has been cancelled, or not.
const std::string data
Data to send, if any.
Definition http_curl.cpp:78
~NetworkHTTPRequest()
Remove ourselves from the callback list.
Definition http_curl.cpp:70
NetworkHTTPRequest(std::string_view uri, HTTPCallback *callback, std::string &&data)
Create a new HTTP request.
Definition http_curl.cpp:60
HTTPThreadSafeCallback callback
Callback to send data back on.
Definition http_curl.cpp:77
const std::string uri
URI to connect to.
Definition http_curl.cpp:76
static void Connect(std::string_view uri, HTTPCallback *callback, std::string &&data="")
Connect to the given URI.
Definition http_curl.cpp:92
static void HTTPReceive()
Do the receiving for all HTTP connections.
Functions related to debugging.
bool IsVisibleSeverity(Facility facility, Severity severity)
Test if debug severity is visible for the given facility.
Definition debug.h:24
#define Debug(facility, severity, format_string,...)
Output a line of debugging information.
Definition debug.h:36
@ Net
Net message facility.
Definition debug_type.h:33
@ Debug1
Debug #1 - High level debug messages.
Definition debug_type.h:20
@ Notice
Notice.
Definition debug_type.h:18
@ Critical
Critical, user should know about this.
Definition debug_type.h:15
@ Debug2
Debug #2 - Low level debug messages.
Definition debug_type.h:21
@ Error
Error, but we are recovering.
Definition debug_type.h:16
bool FileExists(std::string_view filename)
Test whether the given filename exists.
Definition fileio.cpp:133
Functions for standard in/out file operations.
Basic functions to send and receive HTTP packets.
constexpr int HTTP_429_TOO_MANY_REQUESTS
HTTP error code for when the client is doing too many requests.
Definition http.h:13
static std::mutex _http_mutex
Mutex to prevent concurrent access _http_requests.
Definition http_curl.cpp:84
static std::queue< std::unique_ptr< NetworkHTTPRequest > > _http_requests
HTTP requests that are currently running.
Definition http_curl.cpp:83
void NetworkHTTPInitialize()
Initialize the HTTP socket handler.
static std::vector< HTTPThreadSafeCallback * > _new_http_callbacks
Callbacks for the request that should be started.
Definition http_curl.cpp:46
static std::thread _http_thread
The thread running the HTTP requests.
Definition http_curl.cpp:81
static std::condition_variable _http_cv
Conditional variable to wake up the HTTP request thread.
Definition http_curl.cpp:85
static std::atomic< bool > _http_thread_exit
Whether to ask the HTTP request thread to stop.
Definition http_curl.cpp:82
static std::mutex _http_callback_mutex
Mutex to prevent concurrent access to _http_callbacks.
Definition http_curl.cpp:47
void CurlSetOption(CURL *curl, auto option, auto value)
Set some specific option and emit debug information upon failure.
static std::mutex _new_http_callback_mutex
Mutex to prevent concurrent access to _new_http_callbacks.
Definition http_curl.cpp:48
void HttpThread()
Thread entry point for the HTTP request thread.
void NetworkHTTPUninitialize()
Uninitialize the HTTP socket handler.
static std::vector< HTTPThreadSafeCallback * > _http_callbacks
Callback for the current requests.
Definition http_curl.cpp:45
Shared functions for implementations of HTTP requests.
static std::vector< HTTPThreadSafeCallback * > _new_http_callbacks
Callbacks for the request that should be started.
static std::mutex _http_callback_mutex
Mutex to prevent concurrent access to _http_callbacks.
static std::mutex _new_http_callback_mutex
Mutex to prevent concurrent access to _new_http_callbacks.
static std::vector< HTTPThreadSafeCallback * > _http_callbacks
Callback for the current requests.
std::string_view GetNetworkRevisionString()
Get the network version string used by this build.
Convert NetworkGameInfo to Packet and back.
A number of safeguards to prevent using unsafe methods.
Definition of base types and functions in a cross-platform compatible way.
Callback for when the HTTP handler has something to tell us.
Definition http.h:16
virtual void OnFailure()=0
An error has occurred and the connection has been closed.
Base of all threads.
bool StartNewThread(std::thread *thr, std::string_view name, TFn &&_Fx, TArgs &&... _Ax)
Start a new thread.
Definition thread.h:47
std::mutex lock
synchronization for playback status fields
Definition win32_m.cpp:35