OpenTTD Source  20241121-master-g67a0fccfad
network_content.cpp
Go to the documentation of this file.
1 /*
2  * This file is part of OpenTTD.
3  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
6  */
7 
10 #include "../stdafx.h"
11 #include "../rev.h"
12 #include "../ai/ai.hpp"
13 #include "../game/game.hpp"
14 #include "../window_func.h"
15 #include "../error.h"
16 #include "../fileio_func.h"
17 #include "../base_media_base.h"
18 #include "../settings_type.h"
19 #include "network_content.h"
20 
21 #include "table/strings.h"
22 
23 #if defined(WITH_ZLIB)
24 #include <zlib.h>
25 #endif
26 
27 #ifdef __EMSCRIPTEN__
28 # include <emscripten.h>
29 #endif
30 
31 #include "../safeguards.h"
32 
33 extern bool HasScenario(const ContentInfo *ci, bool md5sum);
34 
37 
39 static bool HasGRFConfig(const ContentInfo *ci, bool md5sum)
40 {
41  return FindGRFConfig(BSWAP32(ci->unique_id), md5sum ? FGCM_EXACT : FGCM_ANY, md5sum ? &ci->md5sum : nullptr) != nullptr;
42 }
43 
51 typedef bool (*HasProc)(const ContentInfo *ci, bool md5sum);
52 
54 {
55  ContentInfo *ci = new ContentInfo();
56  ci->type = (ContentType)p.Recv_uint8();
57  ci->id = (ContentID)p.Recv_uint32();
58  ci->filesize = p.Recv_uint32();
59 
64 
65  ci->unique_id = p.Recv_uint32();
66  p.Recv_bytes(ci->md5sum);
67 
68  uint dependency_count = p.Recv_uint8();
69  ci->dependencies.reserve(dependency_count);
70  for (uint i = 0; i < dependency_count; i++) {
71  ContentID dependency_cid = (ContentID)p.Recv_uint32();
72  ci->dependencies.push_back(dependency_cid);
73  this->reverse_dependency_map.emplace(dependency_cid, ci->id);
74  }
75 
76  uint tag_count = p.Recv_uint8();
77  ci->tags.reserve(tag_count);
78  for (uint i = 0; i < tag_count; i++) ci->tags.push_back(p.Recv_string(NETWORK_CONTENT_TAG_LENGTH));
79 
80  if (!ci->IsValid()) {
81  delete ci;
82  this->CloseConnection();
83  return false;
84  }
85 
86  /* Find the appropriate check function */
87  HasProc proc = nullptr;
88  switch (ci->type) {
90  proc = HasGRFConfig;
91  break;
92 
94  proc = BaseGraphics::HasSet;
95  break;
96 
98  proc = BaseMusic::HasSet;
99  break;
100 
102  proc = BaseSounds::HasSet;
103  break;
104 
105  case CONTENT_TYPE_AI:
106  proc = AI::HasAI; break;
107  break;
108 
110  proc = AI::HasAILibrary; break;
111  break;
112 
113  case CONTENT_TYPE_GAME:
114  proc = Game::HasGame; break;
115  break;
116 
118  proc = Game::HasGameLibrary; break;
119  break;
120 
123  proc = HasScenario;
124  break;
125 
126  default:
127  break;
128  }
129 
130  if (proc != nullptr) {
131  if (proc(ci, true)) {
133  } else {
135  if (proc(ci, false)) ci->upgrade = true;
136  }
137  } else {
139  }
140 
141  /* Something we don't have and has filesize 0 does not exist in the system */
143 
144  /* Do we already have a stub for this? */
145  for (ContentInfo *ici : this->infos) {
146  if (ici->type == ci->type && ici->unique_id == ci->unique_id && ci->md5sum == ici->md5sum) {
147  /* Preserve the name if possible */
148  if (ci->name.empty()) ci->name = ici->name;
149  if (ici->IsSelected()) ci->state = ici->state;
150 
151  /*
152  * As ici might be selected by the content window we cannot delete that.
153  * However, we want to keep most of the values of ci, except the values
154  * we (just) already preserved.
155  */
156  *ici = *ci;
157  delete ci;
158 
159  this->OnReceiveContentInfo(ici);
160  return true;
161  }
162  }
163 
164  /* Missing content info? Don't list it */
165  if (ci->filesize == 0) {
166  delete ci;
167  return true;
168  }
169 
170  this->infos.push_back(ci);
171 
172  /* Incoming data means that we might need to reconsider dependencies */
173  ConstContentVector parents;
174  this->ReverseLookupTreeDependency(parents, ci);
175  for (const ContentInfo *ici : parents) {
176  this->CheckDependencyState(const_cast<ContentInfo *>(ici));
177  }
178 
179  this->OnReceiveContentInfo(ci);
180 
181  return true;
182 }
183 
189 {
190  if (type == CONTENT_TYPE_END) {
201  return;
202  }
203 
204  this->Connect();
205 
206  auto p = std::make_unique<Packet>(this, PACKET_CONTENT_CLIENT_INFO_LIST);
207  p->Send_uint8 ((uint8_t)type);
208  p->Send_uint32(0xffffffff);
209  p->Send_uint8 (1);
210  p->Send_string("vanilla");
211  p->Send_string(_openttd_content_version);
212 
213  /* Patchpacks can extend the list with one. In BaNaNaS metadata you can
214  * add a branch in the 'compatibility' list, to filter on this. If you want
215  * your patchpack to be mentioned in the BaNaNaS web-interface, create an
216  * issue on https://github.com/OpenTTD/bananas-api asking for this.
217 
218  p->Send_string("patchpack"); // Or what-ever the name of your patchpack is.
219  p->Send_string(_openttd_content_version_patchpack);
220 
221  */
222 
223  this->SendPacket(std::move(p));
224 }
225 
232 {
233  this->Connect();
234 
235  while (count > 0) {
236  /* We can "only" send a limited number of IDs in a single packet.
237  * A packet begins with the packet size and a byte for the type.
238  * Then this packet adds a uint16_t for the count in this packet.
239  * The rest of the packet can be used for the IDs. */
240  uint p_count = std::min<uint>(count, (TCP_MTU - sizeof(PacketSize) - sizeof(uint8_t) - sizeof(uint16_t)) / sizeof(uint32_t));
241 
242  auto p = std::make_unique<Packet>(this, PACKET_CONTENT_CLIENT_INFO_ID, TCP_MTU);
243  p->Send_uint16(p_count);
244 
245  for (uint i = 0; i < p_count; i++) {
246  p->Send_uint32(content_ids[i]);
247  }
248 
249  this->SendPacket(std::move(p));
250  count -= p_count;
251  content_ids += p_count;
252  }
253 }
254 
261 {
262  if (cv == nullptr) return;
263 
264  this->Connect();
265 
266  assert(cv->size() < 255);
267  assert(cv->size() < (TCP_MTU - sizeof(PacketSize) - sizeof(uint8_t) - sizeof(uint8_t)) /
268  (sizeof(uint8_t) + sizeof(uint32_t) + (send_md5sum ? MD5_HASH_BYTES : 0)));
269 
270  auto p = std::make_unique<Packet>(this, send_md5sum ? PACKET_CONTENT_CLIENT_INFO_EXTID_MD5 : PACKET_CONTENT_CLIENT_INFO_EXTID, TCP_MTU);
271  p->Send_uint8((uint8_t)cv->size());
272 
273  for (const ContentInfo *ci : *cv) {
274  p->Send_uint8((uint8_t)ci->type);
275  p->Send_uint32(ci->unique_id);
276  if (!send_md5sum) continue;
277  p->Send_bytes(ci->md5sum);
278  }
279 
280  this->SendPacket(std::move(p));
281 
282  for (ContentInfo *ci : *cv) {
283  bool found = false;
284  for (ContentInfo *ci2 : this->infos) {
285  if (ci->type == ci2->type && ci->unique_id == ci2->unique_id &&
286  (!send_md5sum || ci->md5sum == ci2->md5sum)) {
287  found = true;
288  break;
289  }
290  }
291  if (!found) {
292  this->infos.push_back(ci);
293  } else {
294  delete ci;
295  }
296  }
297 }
298 
305 void ClientNetworkContentSocketHandler::DownloadSelectedContent(uint &files, uint &bytes, bool fallback)
306 {
307  bytes = 0;
308 
309  ContentIDList content;
310  for (const ContentInfo *ci : this->infos) {
311  if (!ci->IsSelected() || ci->state == ContentInfo::ALREADY_HERE) continue;
312 
313  content.push_back(ci->id);
314  bytes += ci->filesize;
315  }
316 
317  files = (uint)content.size();
318 
319  /* If there's nothing to download, do nothing. */
320  if (files == 0) return;
321 
322  this->isCancelled = false;
323 
324  if (fallback) {
325  this->DownloadSelectedContentFallback(content);
326  } else {
327  this->DownloadSelectedContentHTTP(content);
328  }
329 }
330 
336 {
337  std::string content_request;
338  for (const ContentID &id : content) {
339  content_request += std::to_string(id) + "\n";
340  }
341 
342  this->http_response_index = -1;
343 
345 }
346 
352 {
353  uint count = (uint)content.size();
354  const ContentID *content_ids = content.data();
355  this->Connect();
356 
357  while (count > 0) {
358  /* We can "only" send a limited number of IDs in a single packet.
359  * A packet begins with the packet size and a byte for the type.
360  * Then this packet adds a uint16_t for the count in this packet.
361  * The rest of the packet can be used for the IDs. */
362  uint p_count = std::min<uint>(count, (TCP_MTU - sizeof(PacketSize) - sizeof(uint8_t) - sizeof(uint16_t)) / sizeof(uint32_t));
363 
364  auto p = std::make_unique<Packet>(this, PACKET_CONTENT_CLIENT_CONTENT, TCP_MTU);
365  p->Send_uint16(p_count);
366 
367  for (uint i = 0; i < p_count; i++) {
368  p->Send_uint32(content_ids[i]);
369  }
370 
371  this->SendPacket(std::move(p));
372  count -= p_count;
373  content_ids += p_count;
374  }
375 }
376 
384 static std::string GetFullFilename(const ContentInfo *ci, bool compressed)
385 {
387  if (dir == NO_DIRECTORY) return {};
388 
389  std::string buf = FioGetDirectory(SP_AUTODOWNLOAD_DIR, dir);
390  buf += ci->filename;
391  buf += compressed ? ".tar.gz" : ".tar";
392 
393  return buf;
394 }
395 
401 static bool GunzipFile(const ContentInfo *ci)
402 {
403 #if defined(WITH_ZLIB)
404  bool ret = true;
405 
406  /* Need to open the file with fopen() to support non-ASCII on Windows. */
407  auto ftmp = FileHandle::Open(GetFullFilename(ci, true), "rb");
408  if (!ftmp.has_value()) return false;
409  /* Duplicate the handle, and close the FILE*, to avoid double-closing the handle later. */
410  int fdup = dup(fileno(*ftmp));
411  gzFile fin = gzdopen(fdup, "rb");
412  ftmp.reset();
413 
414  auto fout = FileHandle::Open(GetFullFilename(ci, false), "wb");
415 
416  if (fin == nullptr || !fout.has_value()) {
417  ret = false;
418  } else {
419  uint8_t buff[8192];
420  for (;;) {
421  int read = gzread(fin, buff, sizeof(buff));
422  if (read == 0) {
423  /* If gzread() returns 0, either the end-of-file has been
424  * reached or an underlying read error has occurred.
425  *
426  * gzeof() can't be used, because:
427  * 1.2.5 - it is safe, 1 means 'everything was OK'
428  * 1.2.3.5, 1.2.4 - 0 or 1 is returned 'randomly'
429  * 1.2.3.3 - 1 is returned for truncated archive
430  *
431  * So we use gzerror(). When proper end of archive
432  * has been reached, then:
433  * errnum == Z_STREAM_END in 1.2.3.3,
434  * errnum == 0 in 1.2.4 and 1.2.5 */
435  int errnum;
436  gzerror(fin, &errnum);
437  if (errnum != 0 && errnum != Z_STREAM_END) ret = false;
438  break;
439  }
440  if (read < 0 || static_cast<size_t>(read) != fwrite(buff, 1, read, *fout)) {
441  /* If gzread() returns -1, there was an error in archive */
442  ret = false;
443  break;
444  }
445  /* DO NOT DO THIS! It will fail to detect broken archive with 1.2.3.3!
446  * if (read < sizeof(buff)) break; */
447  }
448  }
449 
450  if (fin != nullptr) {
451  gzclose(fin);
452  } else if (fdup != -1) {
453  /* Failing gzdopen does not close the passed file descriptor. */
454  close(fdup);
455  }
456 
457  return ret;
458 #else
459  NOT_REACHED();
460 #endif /* defined(WITH_ZLIB) */
461 }
462 
470 static inline ssize_t TransferOutFWrite(std::optional<FileHandle> &file, const char *buffer, size_t amount)
471 {
472  return fwrite(buffer, 1, amount, *file);
473 }
474 
476 {
477  if (!this->curFile.has_value()) {
478  delete this->curInfo;
479  /* When we haven't opened a file this must be our first packet with metadata. */
480  this->curInfo = new ContentInfo;
481  this->curInfo->type = (ContentType)p.Recv_uint8();
482  this->curInfo->id = (ContentID)p.Recv_uint32();
483  this->curInfo->filesize = p.Recv_uint32();
485 
486  if (!this->BeforeDownload()) {
487  this->CloseConnection();
488  return false;
489  }
490  } else {
491  /* We have a file opened, thus are downloading internal content */
492  size_t toRead = p.RemainingBytesToTransfer();
493  if (toRead != 0 && static_cast<size_t>(p.TransferOut(TransferOutFWrite, std::ref(this->curFile))) != toRead) {
495  ShowErrorMessage(STR_CONTENT_ERROR_COULD_NOT_DOWNLOAD, STR_CONTENT_ERROR_COULD_NOT_DOWNLOAD_FILE_NOT_WRITABLE, WL_ERROR);
496  this->CloseConnection();
497  this->curFile.reset();
498 
499  return false;
500  }
501 
502  this->OnDownloadProgress(this->curInfo, (int)toRead);
503 
504  if (toRead == 0) this->AfterDownload();
505  }
506 
507  return true;
508 }
509 
515 {
516  if (!this->curInfo->IsValid()) {
517  delete this->curInfo;
518  this->curInfo = nullptr;
519  return false;
520  }
521 
522  if (this->curInfo->filesize != 0) {
523  /* The filesize is > 0, so we are going to download it */
524  std::string filename = GetFullFilename(this->curInfo, true);
525  if (filename.empty() || !(this->curFile = FileHandle::Open(filename, "wb")).has_value()) {
526  /* Unless that fails of course... */
528  ShowErrorMessage(STR_CONTENT_ERROR_COULD_NOT_DOWNLOAD, STR_CONTENT_ERROR_COULD_NOT_DOWNLOAD_FILE_NOT_WRITABLE, WL_ERROR);
529  return false;
530  }
531  }
532  return true;
533 }
534 
540 {
541  /* We read nothing; that's our marker for end-of-stream.
542  * Now gunzip the tar and make it known. */
543  this->curFile.reset();
544 
545  if (GunzipFile(this->curInfo)) {
546  FioRemove(GetFullFilename(this->curInfo, true));
547 
549  if (sd == NO_DIRECTORY) NOT_REACHED();
550 
551  TarScanner ts;
552  std::string fname = GetFullFilename(this->curInfo, false);
553  ts.AddFile(sd, fname);
554 
555  if (this->curInfo->type == CONTENT_TYPE_BASE_MUSIC) {
556  /* Music can't be in a tar. So extract the tar! */
557  ExtractTar(fname, BASESET_DIR);
558  FioRemove(fname);
559  }
560 
561 #ifdef __EMSCRIPTEN__
562  EM_ASM(if (window["openttd_syncfs"]) openttd_syncfs());
563 #endif
564 
565  this->OnDownloadComplete(this->curInfo->id);
566  } else {
567  ShowErrorMessage(STR_CONTENT_ERROR_COULD_NOT_EXTRACT, INVALID_STRING_ID, WL_ERROR);
568  }
569 }
570 
572 {
573  return this->isCancelled;
574 }
575 
576 /* Also called to just clean up the mess. */
578 {
579  this->http_response.clear();
580  this->http_response.shrink_to_fit();
581  this->http_response_index = -2;
582 
583  if (this->curFile.has_value()) {
584  this->OnDownloadProgress(this->curInfo, -1);
585 
586  this->curFile.reset();
587  }
588 
589  /* If we fail, download the rest via the 'old' system. */
590  if (!this->isCancelled) {
591  uint files, bytes;
592 
593  this->DownloadSelectedContent(files, bytes, true);
594  }
595 }
596 
597 void ClientNetworkContentSocketHandler::OnReceiveData(std::unique_ptr<char[]> data, size_t length)
598 {
599  assert(data.get() == nullptr || length != 0);
600 
601  /* Ignore any latent data coming from a connection we closed. */
602  if (this->http_response_index == -2) {
603  return;
604  }
605 
606  if (this->http_response_index == -1) {
607  if (data != nullptr) {
608  /* Append the rest of the response. */
609  this->http_response.insert(this->http_response.end(), data.get(), data.get() + length);
610  return;
611  } else {
612  /* Make sure the response is properly terminated. */
613  this->http_response.push_back('\0');
614 
615  /* And prepare for receiving the rest of the data. */
616  this->http_response_index = 0;
617  }
618  }
619 
620  if (data != nullptr) {
621  /* We have data, so write it to the file. */
622  if (fwrite(data.get(), 1, length, *this->curFile) != length) {
623  /* Writing failed somehow, let try via the old method. */
624  this->OnFailure();
625  } else {
626  /* Just received the data. */
627  this->OnDownloadProgress(this->curInfo, (int)length);
628  }
629 
630  /* Nothing more to do now. */
631  return;
632  }
633 
634  if (this->curFile.has_value()) {
635  /* We've finished downloading a file. */
636  this->AfterDownload();
637  }
638 
639  if ((uint)this->http_response_index >= this->http_response.size()) {
640  /* It's not a real failure, but if there's
641  * nothing more to download it helps with
642  * cleaning up the stuff we allocated. */
643  this->OnFailure();
644  return;
645  }
646 
647  delete this->curInfo;
648  /* When we haven't opened a file this must be our first packet with metadata. */
649  this->curInfo = new ContentInfo;
650 
652 #define check_not_null(p) { if ((p) == nullptr) { this->OnFailure(); return; } }
654 #define check_and_terminate(p) { check_not_null(p); *(p) = '\0'; }
655 
656  for (;;) {
657  char *str = this->http_response.data() + this->http_response_index;
658  char *p = strchr(str, '\n');
659  check_and_terminate(p);
660 
661  /* Update the index for the next one */
662  this->http_response_index += (int)strlen(str) + 1;
663 
664  /* Read the ID */
665  p = strchr(str, ',');
666  check_and_terminate(p);
667  this->curInfo->id = (ContentID)atoi(str);
668 
669  /* Read the type */
670  str = p + 1;
671  p = strchr(str, ',');
672  check_and_terminate(p);
673  this->curInfo->type = (ContentType)atoi(str);
674 
675  /* Read the file size */
676  str = p + 1;
677  p = strchr(str, ',');
678  check_and_terminate(p);
679  this->curInfo->filesize = atoi(str);
680 
681  /* Read the URL */
682  str = p + 1;
683  /* Is it a fallback URL? If so, just continue with the next one. */
684  if (strncmp(str, "ottd", 4) == 0) {
685  if ((uint)this->http_response_index >= this->http_response.size()) {
686  /* Have we gone through all lines? */
687  this->OnFailure();
688  return;
689  }
690  continue;
691  }
692 
693  p = strrchr(str, '/');
694  check_not_null(p);
695  p++; // Start after the '/'
696 
697  std::string filename = p;
698  /* Remove the extension from the string. */
699  for (uint i = 0; i < 2; i++) {
700  auto pos = filename.find_last_of('.');
701  if (pos == std::string::npos) {
702  this->OnFailure();
703  return;
704  }
705  filename.erase(pos);
706  }
707 
708  /* Copy the string, without extension, to the filename. */
709  this->curInfo->filename = std::move(filename);
710 
711  /* Request the next file. */
712  if (!this->BeforeDownload()) {
713  this->OnFailure();
714  return;
715  }
716 
718  return;
719  }
720 
721 #undef check
722 #undef check_and_terminate
723 }
724 
730  http_response_index(-2),
731  curFile(std::nullopt),
732  curInfo(nullptr),
733  isConnecting(false),
734  isCancelled(false)
735 {
736  this->lastActivity = std::chrono::steady_clock::now();
737 }
738 
741 {
742  delete this->curInfo;
743 
744  for (ContentInfo *ci : this->infos) delete ci;
745 }
746 
749 public:
755 
756  void OnFailure() override
757  {
759  _network_content_client.OnConnect(false);
760  }
761 
762  void OnConnect(SOCKET s) override
763  {
764  assert(_network_content_client.sock == INVALID_SOCKET);
765  _network_content_client.lastActivity = std::chrono::steady_clock::now();
769  _network_content_client.OnConnect(true);
770  }
771 };
772 
777 {
778  if (this->sock != INVALID_SOCKET || this->isConnecting) return;
779 
780  this->isCancelled = false;
781  this->isConnecting = true;
782 
783  TCPConnecter::Create<NetworkContentConnecter>(NetworkContentServerConnectionString());
784 }
785 
790 {
792 
793  if (this->sock == INVALID_SOCKET) return NETWORK_RECV_STATUS_OKAY;
794 
795  this->CloseSocket();
796  this->OnDisconnect();
797 
799 }
800 
805 {
806  this->isCancelled = true;
807  this->CloseConnection();
808 }
809 
815 {
816  if (this->sock == INVALID_SOCKET || this->isConnecting) return;
817 
818  /* Close the connection to the content server after inactivity; there can still be downloads pending via HTTP. */
819  if (std::chrono::steady_clock::now() > this->lastActivity + IDLE_TIMEOUT) {
820  this->CloseConnection();
821  return;
822  }
823 
824  if (this->CanSendReceive()) {
825  if (this->ReceivePackets()) {
826  /* Only update activity once a packet is received, instead of every time we try it. */
827  this->lastActivity = std::chrono::steady_clock::now();
828  }
829  }
830 
831  this->SendPackets();
832 }
833 
839 {
840  /* When we tried to download it already, don't try again */
841  if (std::find(this->requested.begin(), this->requested.end(), cid) != this->requested.end()) return;
842 
843  this->requested.push_back(cid);
844  this->RequestContentList(1, &cid);
845 }
846 
853 {
854  for (ContentInfo *ci : this->infos) {
855  if (ci->id == cid) return ci;
856  }
857  return nullptr;
858 }
859 
860 
866 {
867  ContentInfo *ci = this->GetContent(cid);
868  if (ci == nullptr || ci->state != ContentInfo::UNSELECTED) return;
869 
871  this->CheckDependencyState(ci);
872 }
873 
879 {
880  ContentInfo *ci = this->GetContent(cid);
881  if (ci == nullptr || !ci->IsSelected()) return;
882 
884  this->CheckDependencyState(ci);
885 }
886 
889 {
890  for (ContentInfo *ci : this->infos) {
891  if (ci->state == ContentInfo::UNSELECTED) {
892  ci->state = ContentInfo::SELECTED;
893  this->CheckDependencyState(ci);
894  }
895  }
896 }
897 
900 {
901  for (ContentInfo *ci : this->infos) {
902  if (ci->state == ContentInfo::UNSELECTED && ci->upgrade) {
903  ci->state = ContentInfo::SELECTED;
904  this->CheckDependencyState(ci);
905  }
906  }
907 }
908 
911 {
912  for (ContentInfo *ci : this->infos) {
913  if (ci->IsSelected() && ci->state != ContentInfo::ALREADY_HERE) ci->state = ContentInfo::UNSELECTED;
914  }
915 }
916 
919 {
920  switch (ci->state) {
923  this->Unselect(ci->id);
924  break;
925 
927  this->Select(ci->id);
928  break;
929 
930  default:
931  break;
932  }
933 }
934 
941 {
942  auto range = this->reverse_dependency_map.equal_range(child->id);
943 
944  for (auto iter = range.first; iter != range.second; ++iter) {
945  parents.push_back(GetContent(iter->second));
946  }
947 }
948 
955 {
956  tree.push_back(child);
957 
958  /* First find all direct parents. We can't use the "normal" iterator as
959  * we are including stuff into the vector and as such the vector's data
960  * store can be reallocated (and thus move), which means out iterating
961  * pointer gets invalid. So fall back to the indices. */
962  for (uint i = 0; i < tree.size(); i++) {
963  ConstContentVector parents;
964  this->ReverseLookupDependency(parents, tree[i]);
965 
966  for (const ContentInfo *ci : parents) {
967  include(tree, ci);
968  }
969  }
970 }
971 
977 {
978  if (ci->IsSelected() || ci->state == ContentInfo::ALREADY_HERE) {
979  /* Selection is easy; just walk all children and set the
980  * autoselected state. That way we can see what we automatically
981  * selected and thus can unselect when a dependency is removed. */
982  for (auto &dependency : ci->dependencies) {
983  ContentInfo *c = this->GetContent(dependency);
984  if (c == nullptr) {
985  this->DownloadContentInfo(dependency);
986  } else if (c->state == ContentInfo::UNSELECTED) {
988  this->CheckDependencyState(c);
989  }
990  }
991  return;
992  }
993 
994  if (ci->state != ContentInfo::UNSELECTED) return;
995 
996  /* For unselection we need to find the parents of us. We need to
997  * unselect them. After that we unselect all children that we
998  * depend on and are not used as dependency for us, but only when
999  * we automatically selected them. */
1000  ConstContentVector parents;
1001  this->ReverseLookupDependency(parents, ci);
1002  for (const ContentInfo *c : parents) {
1003  if (!c->IsSelected()) continue;
1004 
1005  this->Unselect(c->id);
1006  }
1007 
1008  for (auto &dependency : ci->dependencies) {
1009  const ContentInfo *c = this->GetContent(dependency);
1010  if (c == nullptr) {
1011  DownloadContentInfo(dependency);
1012  continue;
1013  }
1014  if (c->state != ContentInfo::AUTOSELECTED) continue;
1015 
1016  /* Only unselect when WE are the only parent. */
1017  parents.clear();
1018  this->ReverseLookupDependency(parents, c);
1019 
1020  /* First check whether anything depends on us */
1021  int sel_count = 0;
1022  bool force_selection = false;
1023  for (const ContentInfo *parent_ci : parents) {
1024  if (parent_ci->IsSelected()) sel_count++;
1025  if (parent_ci->state == ContentInfo::SELECTED) force_selection = true;
1026  }
1027  if (sel_count == 0) {
1028  /* Nothing depends on us */
1029  this->Unselect(c->id);
1030  continue;
1031  }
1032  /* Something manually selected depends directly on us */
1033  if (force_selection) continue;
1034 
1035  /* "Flood" search to find all items in the dependency graph*/
1036  parents.clear();
1037  this->ReverseLookupTreeDependency(parents, c);
1038 
1039  /* Is there anything that is "force" selected?, if so... we're done. */
1040  for (const ContentInfo *parent_ci : parents) {
1041  if (parent_ci->state != ContentInfo::SELECTED) continue;
1042 
1043  force_selection = true;
1044  break;
1045  }
1046 
1047  /* So something depended directly on us */
1048  if (force_selection) continue;
1049 
1050  /* Nothing depends on us, mark the whole graph as unselected.
1051  * After that's done run over them once again to test their children
1052  * to unselect. Don't do it immediately because it'll do exactly what
1053  * we're doing now. */
1054  for (const ContentInfo *parent : parents) {
1055  if (parent->state == ContentInfo::AUTOSELECTED) this->Unselect(parent->id);
1056  }
1057  for (const ContentInfo *parent : parents) {
1058  this->CheckDependencyState(this->GetContent(parent->id));
1059  }
1060  }
1061 }
1062 
1065 {
1066  for (ContentInfo *c : this->infos) delete c;
1067 
1068  this->infos.clear();
1069  this->requested.clear();
1070  this->reverse_dependency_map.clear();
1071 }
1072 
1073 /*** CALLBACK ***/
1074 
1075 void ClientNetworkContentSocketHandler::OnConnect(bool success)
1076 {
1077  for (size_t i = 0; i < this->callbacks.size(); /* nothing */) {
1078  ContentCallback *cb = this->callbacks[i];
1079  /* the callback may remove itself from this->callbacks */
1080  cb->OnConnect(success);
1081  if (i != this->callbacks.size() && this->callbacks[i] == cb) i++;
1082  }
1083 }
1084 
1086 {
1087  for (size_t i = 0; i < this->callbacks.size(); /* nothing */) {
1088  ContentCallback *cb = this->callbacks[i];
1089  cb->OnDisconnect();
1090  if (i != this->callbacks.size() && this->callbacks[i] == cb) i++;
1091  }
1092 }
1093 
1094 void ClientNetworkContentSocketHandler::OnReceiveContentInfo(const ContentInfo *ci)
1095 {
1096  for (size_t i = 0; i < this->callbacks.size(); /* nothing */) {
1097  ContentCallback *cb = this->callbacks[i];
1098  /* the callback may add items and/or remove itself from this->callbacks */
1099  cb->OnReceiveContentInfo(ci);
1100  if (i != this->callbacks.size() && this->callbacks[i] == cb) i++;
1101  }
1102 }
1103 
1104 void ClientNetworkContentSocketHandler::OnDownloadProgress(const ContentInfo *ci, int bytes)
1105 {
1106  for (size_t i = 0; i < this->callbacks.size(); /* nothing */) {
1107  ContentCallback *cb = this->callbacks[i];
1108  cb->OnDownloadProgress(ci, bytes);
1109  if (i != this->callbacks.size() && this->callbacks[i] == cb) i++;
1110  }
1111 }
1112 
1113 void ClientNetworkContentSocketHandler::OnDownloadComplete(ContentID cid)
1114 {
1115  ContentInfo *ci = this->GetContent(cid);
1116  if (ci != nullptr) {
1118  }
1119 
1120  for (size_t i = 0; i < this->callbacks.size(); /* nothing */) {
1121  ContentCallback *cb = this->callbacks[i];
1122  /* the callback may remove itself from this->callbacks */
1123  cb->OnDownloadComplete(cid);
1124  if (i != this->callbacks.size() && this->callbacks[i] == cb) i++;
1125  }
1126 }
static uint32_t BSWAP32(uint32_t x)
Perform a 32 bits endianness bitswap on x.
static bool HasAI(const struct ContentInfo *ci, bool md5sum)
Wrapper function for AIScanner::HasAI.
Definition: ai_core.cpp:345
static bool HasSet(const ContentInfo *ci, bool md5sum)
Check whether we have an set with the exact characteristics as ci.
Socket handler for the content server connection.
void SelectUpgrade()
Select everything that's an update for something we've got.
void DownloadSelectedContentHTTP(const ContentIDList &content)
Initiate downloading the content over HTTP.
std::chrono::steady_clock::time_point lastActivity
The last time there was network activity.
void DownloadSelectedContent(uint &files, uint &bytes, bool fallback=false)
Actually begin downloading the content we selected.
bool isCancelled
Whether the download has been cancelled.
void ToggleSelectedState(const ContentInfo *ci)
Toggle the state of a content info and check its dependencies.
ClientNetworkContentSocketHandler()
Create a socket handler to handle the connection.
bool isConnecting
Whether we're connecting.
static constexpr std::chrono::seconds IDLE_TIMEOUT
The idle timeout; when to close the connection because it's idle.
void OnReceiveData(std::unique_ptr< char[]> data, size_t length) override
We're receiving data.
void Select(ContentID cid)
Select a specific content id.
int http_response_index
Where we are, in the response, with handling it.
NetworkRecvStatus CloseConnection(bool error=true) override
Disconnect from the content server.
ContentIDList requested
ContentIDs we already requested (so we don't do it again)
void UnselectAll()
Unselect everything that we've not downloaded so far.
void RequestContentList(ContentType type)
Request the content list for the given type.
std::vector< char > http_response
The HTTP response to the requests we've been doing.
void ReverseLookupTreeDependency(ConstContentVector &tree, const ContentInfo *child) const
Reverse lookup the dependencies of all parents over a given child.
void DownloadContentInfo(ContentID cid)
Download information of a given Content ID if not already tried.
bool Receive_SERVER_INFO(Packet &p) override
Server sending list of content info: uint8_t type (invalid ID == does not exist) uint32_t id uint32_t...
void SendReceive()
Check whether we received/can send some data from/to the content server and when that's the case hand...
ContentVector infos
All content info we received.
void OnFailure() override
An error has occurred and the connection has been closed.
ContentInfo * GetContent(ContentID cid) const
Get the content info based on a ContentID.
bool Receive_SERVER_CONTENT(Packet &p) override
Server sending list of content info: uint32_t unique id uint32_t file size (0 == does not exist) stri...
void Connect()
Connect with the content server.
std::unordered_multimap< ContentID, ContentID > reverse_dependency_map
Content reverse dependency map.
bool BeforeDownload()
Handle the opening of the file before downloading.
void Clear()
Clear all downloaded content information.
ContentInfo * curInfo
Information about the currently downloaded file.
std::optional< FileHandle > curFile
Currently downloaded file.
void CheckDependencyState(ContentInfo *ci)
Check the dependencies (recursively) of this content info.
std::vector< ContentID > ContentIDList
List of content IDs to (possibly) select.
void AfterDownload()
Handle the closing and extracting of a file after downloading it has been done.
bool IsCancelled() const override
Check if there is a request to cancel the transfer.
std::vector< ContentCallback * > callbacks
Callbacks to notify "the world".
~ClientNetworkContentSocketHandler()
Clear up the mess ;)
void DownloadSelectedContentFallback(const ContentIDList &content)
Initiate downloading the content over the fallback protocol.
void Unselect(ContentID cid)
Unselect a specific content id.
void Cancel()
Cancel the current download.
void OnDisconnect() override
Callback for when the connection got disconnected.
void ReverseLookupDependency(ConstContentVector &parents, const ContentInfo *child) const
Reverse lookup the dependencies of (direct) parents over a given child.
void SelectAll()
Select everything we can select.
static std::optional< FileHandle > Open(const std::string &filename, const std::string &mode)
Open an RAII file handle if possible.
Definition: fileio.cpp:1170
static bool HasGame(const struct ContentInfo *ci, bool md5sum)
Wrapper function for GameScanner::HasGame.
Definition: game_core.cpp:253
Connect to the content server.
void OnFailure() override
Callback for when the connection attempt failed.
NetworkContentConnecter(const std::string &connection_string)
Initiate the connecting.
Base socket handler for all Content TCP sockets.
Definition: tcp_content.h:22
bool ReceivePackets()
Receive a packet at TCP level.
static void Connect(const std::string &uri, HTTPCallback *callback, const std::string data="")
Connect to the given URI.
Definition: http_curl.cpp:93
void Reopen()
Reopen the socket so we can send/receive stuff again.
Definition: core.h:79
virtual NetworkRecvStatus CloseConnection(bool error=true)
This will put this socket handler in a close state.
Definition: tcp.cpp:51
SOCKET sock
The socket currently connected to.
Definition: tcp.h:38
virtual void SendPacket(std::unique_ptr< Packet > &&packet)
This function puts the packet in the send-queue and it is send as soon as possible.
Definition: tcp.cpp:68
SendPacketsState SendPackets(bool closing_down=false)
Sends all the buffered packets out for this client.
Definition: tcp.cpp:86
void CloseSocket()
Close the actual socket of the connection.
Definition: tcp.cpp:39
bool CanSendReceive()
Check whether this socket can send or receive something.
Definition: tcp.cpp:204
"Helper" class for creating TCP connections in a non-blocking manner
Definition: tcp.h:70
std::string connection_string
Current address we are connecting to (before resolving).
Definition: tcp.h:99
Helper for scanning for files with tar as extension.
Definition: fileio_func.h:59
bool AddFile(const std::string &filename, size_t basepath_length, const std::string &tar_filename={}) override
Add a file with the given filename.
const char * NetworkContentMirrorUriString()
Get the URI string for the content mirror from the environment variable OTTD_CONTENT_MIRROR_URI,...
Definition: config.cpp:65
const char * NetworkContentServerConnectionString()
Get the connection string for the content server from the environment variable OTTD_CONTENT_SERVER_CS...
Definition: config.cpp:55
static const uint16_t NETWORK_CONTENT_SERVER_PORT
The default port of the content server (TCP)
Definition: config.h:24
static const uint NETWORK_CONTENT_URL_LENGTH
The maximum length of a content's url, in bytes including '\0'.
Definition: config.h:65
static const uint NETWORK_CONTENT_VERSION_LENGTH
The maximum length of a content's version, in bytes including '\0'.
Definition: config.h:64
static const size_t TCP_MTU
Number of bytes we can pack in a single TCP packet.
Definition: config.h:45
static const uint NETWORK_CONTENT_FILENAME_LENGTH
The maximum length of a content's filename, in bytes including '\0'.
Definition: config.h:62
static const uint NETWORK_CONTENT_DESC_LENGTH
The maximum length of a content's description, in bytes including '\0'.
Definition: config.h:66
static const uint NETWORK_CONTENT_TAG_LENGTH
The maximum length of a content's tag, in bytes including '\0'.
Definition: config.h:67
static const uint NETWORK_CONTENT_NAME_LENGTH
The maximum length of a content's name, in bytes including '\0'.
Definition: config.h:63
bool include(Container &container, typename Container::const_reference &item)
Helper function to append an item to a container if it is not already contained.
NetworkRecvStatus
Status of a network client; reasons why a client has quit.
Definition: core.h:23
@ NETWORK_RECV_STATUS_OKAY
Everything is okay.
Definition: core.h:24
void ShowErrorMessage(StringID summary_msg, int x, int y, CommandCost cc)
Display an error message in a window.
Definition: error_gui.cpp:367
@ WL_ERROR
Errors (eg. saving/loading failed)
Definition: error.h:26
bool ExtractTar(const std::string &tar_filename, Subdirectory subdir)
Extract the tar with the given filename in the directory where the tar resides.
Definition: fileio.cpp:590
bool FioRemove(const std::string &filename)
Remove a file.
Definition: fileio.cpp:328
@ SP_AUTODOWNLOAD_DIR
Search within the autodownload directory.
Definition: fileio_type.h:150
Subdirectory
The different kinds of subdirectories OpenTTD uses.
Definition: fileio_type.h:115
@ NO_DIRECTORY
A path without any base directory.
Definition: fileio_type.h:133
@ BASESET_DIR
Subdirectory for all base data (base sets, intro game)
Definition: fileio_type.h:123
bool HasScenario(const ContentInfo *ci, bool md5sum)
Check whether we've got a given scenario based on its unique ID.
Definition: fios.cpp:710
static bool HasGRFConfig(const ContentInfo *ci, bool md5sum)
Wrapper function for the HasProc.
static ssize_t TransferOutFWrite(std::optional< FileHandle > &file, const char *buffer, size_t amount)
Simple wrapper around fwrite to be able to pass it to Packet's TransferOut.
ClientNetworkContentSocketHandler _network_content_client
The client we use to connect to the server.
static bool GunzipFile(const ContentInfo *ci)
Gunzip a given file and remove the .gz if successful.
bool(* HasProc)(const ContentInfo *ci, bool md5sum)
Check whether a function piece of content is locally known.
static std::string GetFullFilename(const ContentInfo *ci, bool compressed)
Determine the full filename of a piece of content information.
Part of the network protocol handling content distribution.
std::vector< const ContentInfo * > ConstContentVector
Vector with constant content info.
std::vector< ContentInfo * > ContentVector
Vector with content info.
const GRFConfig * FindGRFConfig(uint32_t grfid, FindGRFConfigMode mode, const MD5Hash *md5sum, uint32_t desired_version)
Find a NewGRF in the scanned list.
@ FGCM_ANY
Use first found.
@ FGCM_EXACT
Only find Grfs matching md5sum.
uint16_t PacketSize
Size of the whole packet.
Definition: packet.h:20
@ SVS_ALLOW_NEWLINE
Allow newlines; replaces '\r ' with ' ' during processing.
Definition: string_type.h:47
@ SVS_REPLACE_WITH_QUESTION_MARK
Replace the unknown/bad bits with question marks.
Definition: string_type.h:46
static const StringID INVALID_STRING_ID
Constant representing an invalid string (16bit in case it is used in savegames)
Definition: strings_type.h:17
Callbacks for notifying others about incoming data.
virtual void OnConnect([[maybe_unused]] bool success)
Callback for when the connection has finished.
virtual void OnDisconnect()
Callback for when the connection got disconnected.
virtual void OnDownloadProgress([[maybe_unused]] const ContentInfo *ci, [[maybe_unused]] int bytes)
We have progress in the download of a file.
virtual void OnDownloadComplete([[maybe_unused]] ContentID cid)
We have finished downloading a file.
virtual void OnReceiveContentInfo([[maybe_unused]] const ContentInfo *ci)
We received a content info.
Container for all important information about a piece of content.
uint32_t unique_id
Unique ID; either GRF ID or shortname.
bool IsValid() const
Is the information from this content info valid?
Definition: tcp_content.cpp:44
uint32_t filesize
Size of the file.
MD5Hash md5sum
The MD5 checksum.
std::string url
URL related to the content.
State state
Whether the content info is selected (for download)
std::string name
Name of the content.
std::string description
Description of the content.
std::string version
Version of the content.
ContentID id
Unique (server side) ID for the content.
@ DOES_NOT_EXIST
The content does not exist in the content system.
@ ALREADY_HERE
The content is already at the client side.
@ AUTOSELECTED
The content has been selected as dependency.
@ UNSELECTED
The content has not been selected.
@ SELECTED
The content has been manually selected.
std::string filename
Filename (for the .tar.gz; only valid on download)
bool IsSelected() const
Is the state either selected or autoselected?
Definition: tcp_content.cpp:27
ContentType type
Type of content.
std::vector< ContentID > dependencies
The dependencies (unique server side ids)
StringList tags
Tags associated with the content.
bool upgrade
This item is an upgrade.
Internal entity of a packet.
Definition: packet.h:42
size_t Recv_bytes(std::span< uint8_t > span)
Extract at most the length of the span bytes from the packet into the span.
Definition: packet.cpp:403
uint32_t Recv_uint32()
Read a 32 bits integer from the packet.
Definition: packet.cpp:347
uint8_t Recv_uint8()
Read a 8 bits integer from the packet.
Definition: packet.cpp:318
size_t RemainingBytesToTransfer() const
Get the amount of bytes that are still available for the Transfer functions.
Definition: packet.cpp:447
ssize_t TransferOut(F transfer_function, D destination, Args &&... args)
Transfer data from the packet to the given function.
Definition: packet.h:139
std::string Recv_string(size_t length, StringValidationSettings settings=SVS_REPLACE_WITH_QUESTION_MARK)
Reads characters (bytes) from the packet until it finds a '\0', or reaches a maximum of length charac...
Definition: packet.cpp:425
Subdirectory GetContentInfoSubDir(ContentType type)
Helper to get the subdirectory a ContentInfo is located in.
ContentType
The values in the enum are important; they are used as database 'keys'.
@ CONTENT_TYPE_AI_LIBRARY
The content consists of an AI library.
@ CONTENT_TYPE_BASE_SOUNDS
The content consists of base sounds.
@ CONTENT_TYPE_GAME_LIBRARY
The content consists of a GS library.
@ CONTENT_TYPE_BASE_GRAPHICS
The content consists of base graphics.
@ CONTENT_TYPE_AI
The content consists of an AI.
@ CONTENT_TYPE_SCENARIO
The content consists of a scenario.
@ CONTENT_TYPE_NEWGRF
The content consists of a NewGRF.
@ CONTENT_TYPE_BASE_MUSIC
The content consists of base music.
@ CONTENT_TYPE_GAME
The content consists of a game script.
@ CONTENT_TYPE_END
Helper to mark the end of the types.
@ CONTENT_TYPE_HEIGHTMAP
The content consists of a heightmap.
ContentID
Unique identifier for the content.
@ PACKET_CONTENT_CLIENT_CONTENT
Request a content file given an internal ID.
@ PACKET_CONTENT_CLIENT_INFO_LIST
Queries the content server for a list of info of a given content type.
@ PACKET_CONTENT_CLIENT_INFO_EXTID_MD5
Queries the content server for information about a list of external IDs and MD5.
@ PACKET_CONTENT_CLIENT_INFO_ID
Queries the content server for information about a list of internal IDs.
@ PACKET_CONTENT_CLIENT_INFO_EXTID
Queries the content server for information about a list of external IDs.
void CloseWindowById(WindowClass cls, WindowNumber number, bool force, int data)
Close a window by its class and window number (if it is open).
Definition: window.cpp:1140
@ WN_NETWORK_STATUS_WINDOW_CONTENT_DOWNLOAD
Network content download status.
Definition: window_type.h:40
@ WC_NETWORK_STATUS_WINDOW
Network status window; Window numbers:
Definition: window_type.h:491