OpenTTD Source  20241108-master-g80f628063a
http_curl.cpp
Go to the documentation of this file.
1 /*
2  * This file is part of OpenTTD.
3  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
6  */
7 
12 #include "../../stdafx.h"
13 #include "../../debug.h"
14 #include "../../fileio_func.h"
15 #include "../../rev.h"
16 #include "../../thread.h"
17 #include "../network_internal.h"
18 
19 #include "http.h"
20 #include "http_shared.h"
21 
22 #include <atomic>
23 #include <condition_variable>
24 #include <curl/curl.h>
25 #include <mutex>
26 #include <queue>
27 
28 #include "../../safeguards.h"
29 
30 #if defined(UNIX)
32 static auto _certificate_files = {
33  "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.
34  "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
35  "/etc/ssl/ca-bundle.pem", // OpenSUSE
36  "/etc/pki/tls/cacert.pem", // OpenELEC
37  "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
38  "/etc/ssl/cert.pem", // Alpine Linux
39 };
41 static auto _certificate_directories = {
42  "/etc/ssl/certs", // SLES10/SLES11, https://golang.org/issue/12139
43  "/etc/pki/tls/certs", // Fedora/RHEL
44  "/system/etc/security/cacerts", // Android
45 };
46 #endif /* UNIX */
47 
48 static std::vector<HTTPThreadSafeCallback *> _http_callbacks;
49 static std::vector<HTTPThreadSafeCallback *> _new_http_callbacks;
50 static std::mutex _http_callback_mutex;
51 static std::mutex _new_http_callback_mutex;
52 
55 public:
63  NetworkHTTPRequest(const std::string &uri, HTTPCallback *callback, const std::string &data) :
64  uri(uri),
66  data(data)
67  {
68  std::lock_guard<std::mutex> lock(_new_http_callback_mutex);
69  _new_http_callbacks.push_back(&this->callback);
70  }
71 
73  {
74  std::lock_guard<std::mutex> lock(_http_callback_mutex);
75  _http_callbacks.erase(std::remove(_http_callbacks.begin(), _http_callbacks.end(), &this->callback), _http_callbacks.end());
76  }
77 
78  const std::string uri;
80  const std::string data;
81 };
82 
83 static std::thread _http_thread;
84 static std::atomic<bool> _http_thread_exit = false;
85 static std::queue<std::unique_ptr<NetworkHTTPRequest>> _http_requests;
86 static std::mutex _http_mutex;
87 static std::condition_variable _http_cv;
88 #if defined(UNIX)
89 static std::string _http_ca_file = "";
90 static std::string _http_ca_path = "";
91 #endif /* UNIX */
92 
93 /* static */ void NetworkHTTPSocketHandler::Connect(const std::string &uri, HTTPCallback *callback, const std::string data)
94 {
95 #if defined(UNIX)
96  if (_http_ca_file.empty() && _http_ca_path.empty()) {
97  callback->OnFailure();
98  return;
99  }
100 #endif /* UNIX */
101 
102  std::lock_guard<std::mutex> lock(_http_mutex);
103  _http_requests.push(std::make_unique<NetworkHTTPRequest>(uri, callback, data));
104  _http_cv.notify_one();
105 }
106 
108 {
109  std::lock_guard<std::mutex> lock(_http_callback_mutex);
110 
111  {
112  std::lock_guard<std::mutex> lock_new(_new_http_callback_mutex);
113  if (!_new_http_callbacks.empty()) {
114  /* We delay adding new callbacks, as HandleQueue() below might add a new callback. */
115  _http_callbacks.insert(_http_callbacks.end(), _new_http_callbacks.begin(), _new_http_callbacks.end());
116  _new_http_callbacks.clear();
117  }
118  }
119 
120  for (auto &callback : _http_callbacks) {
121  callback->HandleQueue();
122  }
123 }
124 
125 void HttpThread()
126 {
127  CURL *curl = curl_easy_init();
128  assert(curl != nullptr);
129 
130  for (;;) {
131  std::unique_lock<std::mutex> lock(_http_mutex);
132 
133  /* Wait for a new request. */
134  while (_http_requests.empty() && !_http_thread_exit) {
135  _http_cv.wait(lock);
136  }
137  if (_http_thread_exit) break;
138 
139  std::unique_ptr<NetworkHTTPRequest> request = std::move(_http_requests.front());
140  _http_requests.pop();
141 
142  /* Release the lock, as we will take a while to process the request. */
143  lock.unlock();
144 
145  /* Reset to default settings. */
146  curl_easy_reset(curl);
147  curl_slist *headers = nullptr;
148 
149  if (_debug_net_level >= 5) {
150  curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
151  }
152 
153  /* Setup some default options. */
154  std::string user_agent = fmt::format("OpenTTD/{}", GetNetworkRevisionString());
155  curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent.c_str());
156  curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
157  curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);
158 
159  /* Ensure we validate the certificate and hostname of the server. */
160 #if defined(UNIX)
161  curl_easy_setopt(curl, CURLOPT_CAINFO, _http_ca_file.empty() ? nullptr : _http_ca_file.c_str());
162  curl_easy_setopt(curl, CURLOPT_CAPATH, _http_ca_path.empty() ? nullptr : _http_ca_path.c_str());
163 #endif /* UNIX */
164  curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2);
165  curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, true);
166 
167  /* Give the connection about 10 seconds to complete. */
168  curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10L);
169 
170  /* Set a buffer of 100KiB, as the default of 16KiB seems a bit small. */
171  curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, 100L * 1024L);
172 
173  /* Fail our call if we don't receive a 2XX return value. */
174  curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
175 
176  /* Prepare POST body and URI. */
177  if (!request->data.empty()) {
178  /* When the payload starts with a '{', it is a JSON payload. */
179  if (request->data.starts_with("{")) {
180  headers = curl_slist_append(headers, "Content-Type: application/json");
181  } else {
182  headers = curl_slist_append(headers, "Content-Type: application/x-www-form-urlencoded");
183  }
184 
185  curl_easy_setopt(curl, CURLOPT_POST, 1L);
186  curl_easy_setopt(curl, CURLOPT_POSTFIELDS, request->data.c_str());
187  curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
188  }
189  curl_easy_setopt(curl, CURLOPT_URL, request->uri.c_str());
190 
191  /* Setup our (C-style) callback function which we pipe back into the callback. */
192  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, +[](char *ptr, size_t size, size_t nmemb, void *userdata) -> size_t {
193  Debug(net, 6, "HTTP callback: {} bytes", size * nmemb);
194  HTTPThreadSafeCallback *callback = static_cast<HTTPThreadSafeCallback *>(userdata);
195 
196  /* Copy the buffer out of CURL. OnReceiveData() will free it when done. */
197  std::unique_ptr<char[]> buffer = std::make_unique<char[]>(size * nmemb);
198  memcpy(buffer.get(), ptr, size * nmemb);
199  callback->OnReceiveData(std::move(buffer), size * nmemb);
200 
201  return size * nmemb;
202  });
203  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &request->callback);
204 
205  /* Create a callback from which we can cancel. Sadly, there is no other
206  * thread-safe way to do this. If the connection went idle, it can take
207  * up to a second before this callback is called. There is little we can
208  * do about this. */
209  curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
210  curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, +[](void *userdata, curl_off_t /*dltotal*/, curl_off_t /*dlnow*/, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/) -> int {
211  const HTTPThreadSafeCallback *callback = static_cast<HTTPThreadSafeCallback *>(userdata);
212  return (callback->cancelled || _http_thread_exit) ? 1 : 0;
213  });
214  curl_easy_setopt(curl, CURLOPT_XFERINFODATA, &request->callback);
215 
216  /* Perform the request. */
217  CURLcode res = curl_easy_perform(curl);
218 
219  curl_slist_free_all(headers);
220 
221  if (res == CURLE_OK) {
222  Debug(net, 1, "HTTP request succeeded");
223  request->callback.OnReceiveData(nullptr, 0);
224  } else {
225  long status_code = 0;
226  curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status_code);
227 
228  /* No need to be verbose about rate limiting. */
229  Debug(net, (request->callback.cancelled || _http_thread_exit || status_code == HTTP_429_TOO_MANY_REQUESTS) ? 1 : 0, "HTTP request failed: status_code: {}, error: {}", status_code, curl_easy_strerror(res));
230  request->callback.OnFailure();
231  }
232 
233  /* Wait till the callback tells us all data is dequeued, or _http_thread_exit has been set. */
234  request->callback.WaitTillEmptyOrCondition([]() -> bool {
235  return _http_thread_exit;
236  });
237  }
238 
239  curl_easy_cleanup(curl);
240 }
241 
243 {
244  curl_global_init(CURL_GLOBAL_DEFAULT);
245 
246 #if defined(UNIX)
247  /* Depending on the Linux distro, certificates can either be in
248  * a bundle or a folder, in a wide range of different locations.
249  * Try to find what location is used by this OS. */
250  for (auto &ca_file : _certificate_files) {
251  if (FileExists(ca_file)) {
252  _http_ca_file = ca_file;
253  break;
254  }
255  }
256  if (_http_ca_file.empty()) {
257  for (auto &ca_path : _certificate_directories) {
258  if (FileExists(ca_path)) {
259  _http_ca_path = ca_path;
260  break;
261  }
262  }
263  }
264  Debug(net, 3, "Using certificate file: {}", _http_ca_file.empty() ? "none" : _http_ca_file);
265  Debug(net, 3, "Using certificate path: {}", _http_ca_path.empty() ? "none" : _http_ca_path);
266 
267  /* Tell the user why HTTPS will not be working. */
268  if (_http_ca_file.empty() && _http_ca_path.empty()) {
269  Debug(net, 0, "No certificate files or directories found, HTTPS will not work!");
270  }
271 #endif /* UNIX */
272 
273  _http_thread_exit = false;
274  StartNewThread(&_http_thread, "ottd:http", &HttpThread);
275 }
276 
278 {
279  _http_thread_exit = true;
280 
281  /* Ensure the callbacks are handled. This is mostly needed as we send
282  * a survey just before close, and that might be pending here. */
284 
285  {
286  std::lock_guard<std::mutex> lock(_http_mutex);
287  _http_cv.notify_one();
288  }
289 
290  if (_http_thread.joinable()) {
291  _http_thread.join();
292  }
293 
294  curl_global_cleanup();
295 }
Converts a HTTPCallback to a Thread-Safe variant.
Definition: http_shared.h:22
void OnReceiveData(std::unique_ptr< char[]> data, size_t length)
Similar to HTTPCallback::OnReceiveData, but thread-safe.
Definition: http_shared.h:48
Single HTTP request.
Definition: http_curl.cpp:54
NetworkHTTPRequest(const std::string &uri, HTTPCallback *callback, const std::string &data)
Create a new HTTP request.
Definition: http_curl.cpp:63
const std::string data
Data to send, if any.
Definition: http_curl.cpp:80
~NetworkHTTPRequest()
Destructor of the HTTP request.
Definition: http_curl.cpp:72
HTTPThreadSafeCallback callback
Callback to send data back on.
Definition: http_curl.cpp:79
const std::string uri
URI to connect to.
Definition: http_curl.cpp:78
static void Connect(const std::string &uri, HTTPCallback *callback, const std::string data="")
Connect to the given URI.
Definition: http_curl.cpp:93
static void HTTPReceive()
Do the receiving for all HTTP connections.
Definition: http_curl.cpp:107
#define Debug(category, level, format_string,...)
Ouptut a line of debugging information.
Definition: debug.h:37
bool FileExists(const std::string &filename)
Test whether the given filename exists.
Definition: fileio.cpp:132
Basic functions to send and receive HTTP packets.
void NetworkHTTPInitialize()
Initialize the HTTP socket handler.
Definition: http_curl.cpp:242
void NetworkHTTPUninitialize()
Uninitialize the HTTP socket handler.
Definition: http_curl.cpp:277
Shared functions for implementations of HTTP requests.
Callback for when the HTTP handler has something to tell us.
Definition: http.h:20
virtual void OnFailure()=0
An error has occurred and the connection has been closed.
bool StartNewThread(std::thread *thr, const char *name, TFn &&_Fx, TArgs &&... _Ax)
Start a new thread.
Definition: thread.h:47
std::mutex lock
synchronization for playback status fields
Definition: win32_m.cpp:35