OpenTTD Source  20241108-master-g80f628063a
strgen.cpp
Go to the documentation of this file.
1 /*
2  * This file is part of OpenTTD.
3  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
6  */
7 
10 #include "../stdafx.h"
11 #include "../core/endian_func.hpp"
12 #include "../core/mem_func.hpp"
13 #include "../error_func.h"
14 #include "../string_func.h"
15 #include "../strings_type.h"
16 #include "../misc/getoptdata.h"
17 #include "../table/control_codes.h"
18 #include "../3rdparty/fmt/std.h"
19 
20 #include "strgen.h"
21 
22 #include <filesystem>
23 #include <fstream>
24 
25 #include "../table/strgen_tables.h"
26 
27 #include "../safeguards.h"
28 
29 
30 #ifdef _MSC_VER
31 # define LINE_NUM_FMT(s) "{} ({}): warning: {} (" s ")\n"
32 #else
33 # define LINE_NUM_FMT(s) "{}:{}: " s ": {}\n"
34 #endif
35 
36 void StrgenWarningI(const std::string &msg)
37 {
38  if (_show_todo > 0) {
39  fmt::print(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, msg);
40  } else {
41  fmt::print(stderr, LINE_NUM_FMT("info"), _file, _cur_line, msg);
42  }
43  _warnings++;
44 }
45 
46 void StrgenErrorI(const std::string &msg)
47 {
48  fmt::print(stderr, LINE_NUM_FMT("error"), _file, _cur_line, msg);
49  _errors++;
50 }
51 
52 [[noreturn]] void StrgenFatalI(const std::string &msg)
53 {
54  fmt::print(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, msg);
55 #ifdef _MSC_VER
56  fmt::print(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
57 #endif
58  throw std::exception();
59 }
60 
61 [[noreturn]] void FatalErrorI(const std::string &msg)
62 {
63  fmt::print(stderr, LINE_NUM_FMT("FATAL"), _file, _cur_line, msg);
64 #ifdef _MSC_VER
65  fmt::print(stderr, LINE_NUM_FMT("warning"), _file, _cur_line, "language is not compiled");
66 #endif
67  exit(2);
68 }
69 
72  std::ifstream input_stream;
73 
81  FileStringReader(StringData &data, const std::filesystem::path &file, bool master, bool translation) :
82  StringReader(data, file.generic_string(), master, translation)
83  {
84  this->input_stream.open(file, std::ifstream::binary);
85  }
86 
87  std::optional<std::string> ReadLine() override
88  {
89  std::string result;
90  if (!std::getline(this->input_stream, result)) return std::nullopt;
91  return result;
92  }
93 
94  void HandlePragma(char *str) override;
95 
96  void ParseFile() override
97  {
99 
101  FatalError("Language must include ##name, ##ownname and ##isocode");
102  }
103  }
104 };
105 
107 {
108  if (!memcmp(str, "id ", 3)) {
109  this->data.next_string_id = std::strtoul(str + 3, nullptr, 0);
110  } else if (!memcmp(str, "name ", 5)) {
111  strecpy(_lang.name, str + 5);
112  } else if (!memcmp(str, "ownname ", 8)) {
113  strecpy(_lang.own_name, str + 8);
114  } else if (!memcmp(str, "isocode ", 8)) {
115  strecpy(_lang.isocode, str + 8);
116  } else if (!memcmp(str, "textdir ", 8)) {
117  if (!memcmp(str + 8, "ltr", 3)) {
119  } else if (!memcmp(str + 8, "rtl", 3)) {
121  } else {
122  FatalError("Invalid textdir {}", str + 8);
123  }
124  } else if (!memcmp(str, "digitsep ", 9)) {
125  str += 9;
126  strecpy(_lang.digit_group_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str);
127  } else if (!memcmp(str, "digitsepcur ", 12)) {
128  str += 12;
129  strecpy(_lang.digit_group_separator_currency, strcmp(str, "{NBSP}") == 0 ? NBSP : str);
130  } else if (!memcmp(str, "decimalsep ", 11)) {
131  str += 11;
132  strecpy(_lang.digit_decimal_separator, strcmp(str, "{NBSP}") == 0 ? NBSP : str);
133  } else if (!memcmp(str, "winlangid ", 10)) {
134  const char *buf = str + 10;
135  long langid = std::strtol(buf, nullptr, 16);
136  if (langid > (long)UINT16_MAX || langid < 0) {
137  FatalError("Invalid winlangid {}", buf);
138  }
139  _lang.winlangid = (uint16_t)langid;
140  } else if (!memcmp(str, "grflangid ", 10)) {
141  const char *buf = str + 10;
142  long langid = std::strtol(buf, nullptr, 16);
143  if (langid >= 0x7F || langid < 0) {
144  FatalError("Invalid grflangid {}", buf);
145  }
146  _lang.newgrflangid = (uint8_t)langid;
147  } else if (!memcmp(str, "gender ", 7)) {
148  if (this->master) FatalError("Genders are not allowed in the base translation.");
149  char *buf = str + 7;
150 
151  for (;;) {
152  const char *s = ParseWord(&buf);
153 
154  if (s == nullptr) break;
155  if (_lang.num_genders >= MAX_NUM_GENDERS) FatalError("Too many genders, max {}", MAX_NUM_GENDERS);
157  _lang.num_genders++;
158  }
159  } else if (!memcmp(str, "case ", 5)) {
160  if (this->master) FatalError("Cases are not allowed in the base translation.");
161  char *buf = str + 5;
162 
163  for (;;) {
164  const char *s = ParseWord(&buf);
165 
166  if (s == nullptr) break;
167  if (_lang.num_cases >= MAX_NUM_CASES) FatalError("Too many cases, max {}", MAX_NUM_CASES);
169  _lang.num_cases++;
170  }
171  } else {
173  }
174 }
175 
176 bool CompareFiles(const std::filesystem::path &path1, const std::filesystem::path &path2)
177 {
178  /* Check for equal size, but ignore the error code for cases when a file does not exist. */
179  std::error_code error_code;
180  if (std::filesystem::file_size(path1, error_code) != std::filesystem::file_size(path2, error_code)) return false;
181 
182  std::ifstream stream1(path1, std::ifstream::binary);
183  std::ifstream stream2(path2, std::ifstream::binary);
184 
185  return std::equal(std::istreambuf_iterator<char>(stream1.rdbuf()),
186  std::istreambuf_iterator<char>(),
187  std::istreambuf_iterator<char>(stream2.rdbuf()));
188 }
189 
191 struct FileWriter {
192  std::ofstream output_stream;
193  const std::filesystem::path path;
194 
200  FileWriter(const std::filesystem::path &path, std::ios_base::openmode openmode) : path(path)
201  {
202  this->output_stream.open(path, openmode);
203  }
204 
206  void Finalise()
207  {
208  this->output_stream.close();
209  }
210 
212  virtual ~FileWriter()
213  {
214  /* If we weren't closed an exception was thrown, so remove the temporary file. */
215  if (this->output_stream.is_open()) {
216  this->output_stream.close();
217  std::filesystem::remove(this->path);
218  }
219  }
220 };
221 
224  const std::filesystem::path real_path;
226  int prev;
227  uint total_strings;
228 
233  HeaderFileWriter(const std::filesystem::path &path) : FileWriter("tmp.xxx", std::ofstream::out),
234  real_path(path), prev(0), total_strings(0)
235  {
236  this->output_stream << "/* This file is automatically generated. Do not modify */\n\n";
237  this->output_stream << "#ifndef TABLE_STRINGS_H\n";
238  this->output_stream << "#define TABLE_STRINGS_H\n";
239  }
240 
241  void WriteStringID(const std::string &name, int stringid) override
242  {
243  if (prev + 1 != stringid) this->output_stream << "\n";
244  fmt::print(this->output_stream, "static const StringID {} = 0x{:X};\n", name, stringid);
245  prev = stringid;
246  total_strings++;
247  }
248 
249  void Finalise(const StringData &data) override
250  {
251  /* Find the plural form with the most amount of cases. */
252  int max_plural_forms = 0;
253  for (const auto &pf : _plural_forms) {
254  max_plural_forms = std::max(max_plural_forms, pf.plural_count);
255  }
256 
257  fmt::print(this->output_stream,
258  "\n"
259  "static const uint LANGUAGE_PACK_VERSION = 0x{:X};\n"
260  "static const uint LANGUAGE_MAX_PLURAL = {};\n"
261  "static const uint LANGUAGE_MAX_PLURAL_FORMS = {};\n"
262  "static const uint LANGUAGE_TOTAL_STRINGS = {};\n"
263  "\n",
264  data.Version(), std::size(_plural_forms), max_plural_forms, total_strings
265  );
266 
267  this->output_stream << "#endif /* TABLE_STRINGS_H */\n";
268 
269  this->FileWriter::Finalise();
270 
271  std::error_code error_code;
272  if (CompareFiles(this->path, this->real_path)) {
273  /* files are equal. tmp.xxx is not needed */
274  std::filesystem::remove(this->path, error_code); // Just ignore the error
275  } else {
276  /* else rename tmp.xxx into filename */
277  std::filesystem::rename(this->path, this->real_path, error_code);
278  if (error_code) FatalError("rename({}, {}) failed: {}", this->path, this->real_path, error_code.message());
279  }
280  }
281 };
282 
289  LanguageFileWriter(const std::filesystem::path &path) : FileWriter(path, std::ofstream::binary | std::ofstream::out)
290  {
291  }
292 
293  void WriteHeader(const LanguagePackHeader *header) override
294  {
295  this->Write((const uint8_t *)header, sizeof(*header));
296  }
297 
298  void Finalise() override
299  {
300  this->output_stream.put(0);
301  this->FileWriter::Finalise();
302  }
303 
304  void Write(const uint8_t *buffer, size_t length) override
305  {
306  this->output_stream.write((const char *)buffer, length);
307  }
308 };
309 
311 static const OptionData _opts[] = {
312  { .type = ODF_NO_VALUE, .id = 'C', .longname = "-export-commands" },
313  { .type = ODF_NO_VALUE, .id = 'L', .longname = "-export-plurals" },
314  { .type = ODF_NO_VALUE, .id = 'P', .longname = "-export-pragmas" },
315  { .type = ODF_NO_VALUE, .id = 't', .shortname = 't', .longname = "--todo" },
316  { .type = ODF_NO_VALUE, .id = 'w', .shortname = 'w', .longname = "--warning" },
317  { .type = ODF_NO_VALUE, .id = 'h', .shortname = 'h', .longname = "--help" },
318  { .type = ODF_NO_VALUE, .id = 'h', .shortname = '?' },
319  { .type = ODF_HAS_VALUE, .id = 's', .shortname = 's', .longname = "--source_dir" },
320  { .type = ODF_HAS_VALUE, .id = 'd', .shortname = 'd', .longname = "--dest_dir" },
321 };
322 
323 int CDECL main(int argc, char *argv[])
324 {
325  std::filesystem::path src_dir(".");
326  std::filesystem::path dest_dir;
327 
328  GetOptData mgo(std::span(argv + 1, argc - 1), _opts);
329  for (;;) {
330  int i = mgo.GetOpt();
331  if (i == -1) break;
332 
333  switch (i) {
334  case 'C':
335  fmt::print("args\tflags\tcommand\treplacement\n");
336  for (const auto &cs : _cmd_structs) {
337  char flags;
338  if (cs.proc == EmitGender) {
339  flags = 'g'; // Command needs number of parameters defined by number of genders
340  } else if (cs.proc == EmitPlural) {
341  flags = 'p'; // Command needs number of parameters defined by plural value
342  } else if (cs.flags & C_DONTCOUNT) {
343  flags = 'i'; // Command may be in the translation when it is not in base
344  } else {
345  flags = '0'; // Command needs no parameters
346  }
347  fmt::print("{}\t{:c}\t\"{}\"\t\"{}\"\n", cs.consumes, flags, cs.cmd, strstr(cs.cmd, "STRING") ? "STRING" : cs.cmd);
348  }
349  return 0;
350 
351  case 'L':
352  fmt::print("count\tdescription\tnames\n");
353  for (const auto &pf : _plural_forms) {
354  fmt::print("{}\t\"{}\"\t{}\n", pf.plural_count, pf.description, pf.names);
355  }
356  return 0;
357 
358  case 'P':
359  fmt::print("name\tflags\tdefault\tdescription\n");
360  for (const auto &pragma : _pragmas) {
361  fmt::print("\"{}\"\t{}\t\"{}\"\t\"{}\"\n",
362  pragma[0], pragma[1], pragma[2], pragma[3]);
363  }
364  return 0;
365 
366  case 't':
367  _show_todo |= 1;
368  break;
369 
370  case 'w':
371  _show_todo |= 2;
372  break;
373 
374  case 'h':
375  fmt::print(
376  "strgen\n"
377  " -t | --todo replace any untranslated strings with '<TODO>'\n"
378  " -w | --warning print a warning for any untranslated strings\n"
379  " -h | -? | --help print this help message and exit\n"
380  " -s | --source_dir search for english.txt in the specified directory\n"
381  " -d | --dest_dir put output file in the specified directory, create if needed\n"
382  " -export-commands export all commands and exit\n"
383  " -export-plurals export all plural forms and exit\n"
384  " -export-pragmas export all pragmas and exit\n"
385  " Run without parameters and strgen will search for english.txt and parse it,\n"
386  " creating strings.h. Passing an argument, strgen will translate that language\n"
387  " file using english.txt as a reference and output <language>.lng.\n"
388  );
389  return 0;
390 
391  case 's':
392  src_dir = mgo.opt;
393  break;
394 
395  case 'd':
396  dest_dir = mgo.opt;
397  break;
398 
399  case -2:
400  fmt::print(stderr, "Invalid arguments\n");
401  return 0;
402  }
403  }
404 
405  if (dest_dir.empty()) dest_dir = src_dir; // if dest_dir is not specified, it equals src_dir
406 
407  try {
408  /* strgen has two modes of operation. If no (free) arguments are passed
409  * strgen generates strings.h to the destination directory. If it is supplied
410  * with a (free) parameter the program will translate that language to destination
411  * directory. As input english.txt is parsed from the source directory */
412  if (mgo.arguments.empty()) {
413  std::filesystem::path input_path = src_dir;
414  input_path /= "english.txt";
415 
416  /* parse master file */
417  StringData data(TEXT_TAB_END);
418  FileStringReader master_reader(data, input_path, true, false);
419  master_reader.ParseFile();
420  if (_errors != 0) return 1;
421 
422  /* write strings.h */
423  std::filesystem::path output_path = dest_dir;
424  std::filesystem::create_directories(dest_dir);
425  output_path /= "strings.h";
426 
427  HeaderFileWriter writer(output_path);
428  writer.WriteHeader(data);
429  writer.Finalise(data);
430  if (_errors != 0) return 1;
431  } else {
432  std::filesystem::path input_path = src_dir;
433  input_path /= "english.txt";
434 
435  StringData data(TEXT_TAB_END);
436  /* parse master file and check if target file is correct */
437  FileStringReader master_reader(data, input_path, true, false);
438  master_reader.ParseFile();
439 
440  for (auto &argument: mgo.arguments) {
441  data.FreeTranslation();
442 
443  std::filesystem::path lang_file = argument;
444  FileStringReader translation_reader(data, lang_file, false, lang_file.filename() != "english.txt");
445  translation_reader.ParseFile(); // target file
446  if (_errors != 0) return 1;
447 
448  /* get the targetfile, strip any directories and append to destination path */
449  std::filesystem::path output_file = dest_dir;
450  output_file /= lang_file.filename();
451  output_file.replace_extension("lng");
452 
453  LanguageFileWriter writer(output_file);
454  writer.WriteLang(data);
455  writer.Finalise();
456 
457  /* if showing warnings, print a summary of the language */
458  if ((_show_todo & 2) != 0) {
459  fmt::print("{} warnings and {} errors for {}\n", _warnings, _errors, output_file);
460  }
461  }
462  }
463  } catch (...) {
464  return 2;
465  }
466 
467  return 0;
468 }
@ ODF_NO_VALUE
A plain option (no value attached to it).
Definition: getoptdata.h:15
@ ODF_HAS_VALUE
An option with a value.
Definition: getoptdata.h:16
static const uint8_t MAX_NUM_GENDERS
Maximum number of supported genders.
Definition: language.h:20
static const uint8_t MAX_NUM_CASES
Maximum number of supported cases.
Definition: language.h:21
static bool CompareFiles(const char *n1, const char *n2)
Compare two files for identity.
int CDECL main(int argc, char *argv[])
And the main program (what else?)
static const OptionData _opts[]
Options of strgen.
Definition: strgen.cpp:311
void FatalErrorI(const std::string &msg)
Error handling for fatal non-user errors.
Definition: strgen.cpp:61
Structures related to strgen.
LanguagePackHeader _lang
Header information about a language.
Definition: strgen_base.cpp:32
int _cur_line
The current line we're parsing in the input file.
Definition: strgen_base.cpp:30
const char * _file
The filename of the input, so we can refer to it in errors/warnings.
Definition: strgen_base.cpp:29
static const char *const _pragmas[][4]
All pragmas used.
static const PluralForm _plural_forms[]
All plural forms used.
@ C_DONTCOUNT
These commands aren't counted for comparison.
Definition: strgen_tables.h:14
void strecpy(std::span< char > dst, std::string_view src)
Copies characters from one buffer to another.
Definition: string.cpp:60
bool StrEmpty(const char *s)
Check if a string buffer is empty.
Definition: string_func.h:57
#define NBSP
A non-breaking space.
Definition: string_type.h:16
@ TD_LTR
Text is written left-to-right by default.
Definition: strings_type.h:23
@ TD_RTL
Text is written right-to-left by default.
Definition: strings_type.h:24
@ TEXT_TAB_END
End of language files.
Definition: strings_type.h:38
A reader that simply reads using fopen.
Definition: strgen.cpp:71
void HandlePragma(char *str) override
Handle the pragma of the file.
Definition: strgen.cpp:106
FileStringReader(StringData &data, const std::filesystem::path &file, bool master, bool translation)
Create the reader.
Definition: strgen.cpp:81
std::optional< std::string > ReadLine() override
Read a single line from the source of strings.
Definition: strgen.cpp:87
void ParseFile() override
Start parsing the file.
Definition: strgen.cpp:96
Yes, simply writing to a file.
Definition: saveload.cpp:2244
std::ofstream output_stream
The stream to write all the output to.
Definition: strgen.cpp:192
virtual ~FileWriter()
Make sure the file is closed.
Definition: strgen.cpp:212
const std::filesystem::path path
The file name we're writing to.
Definition: strgen.cpp:193
FileWriter(const std::filesystem::path &path, std::ios_base::openmode openmode)
Open a file to write to.
Definition: strgen.cpp:200
void Finalise()
Finalise the writing.
Definition: strgen.cpp:206
Data storage for parsing command line options.
Definition: getoptdata.h:29
const std::filesystem::path real_path
The real path we eventually want to write to.
Definition: strgen.cpp:224
void WriteStringID(const std::string &name, int stringid) override
Write the string ID.
Definition: strgen.cpp:241
void Finalise(const StringData &data) override
Finalise writing the file.
Definition: strgen.cpp:249
HeaderFileWriter(const std::filesystem::path &path)
Open a file to write to.
Definition: strgen.cpp:233
int prev
The previous string ID that was printed.
Definition: strgen.cpp:226
Base class for writing the header, i.e.
Definition: strgen.h:87
Class for writing a language to disk.
Definition: strgen.cpp:284
LanguageFileWriter(const std::filesystem::path &path)
Open a file to write to.
Definition: strgen.cpp:289
void Finalise() override
Finalise writing the file.
Definition: strgen.cpp:298
void WriteHeader(const LanguagePackHeader *header) override
Write the header metadata.
Definition: strgen.cpp:293
void Write(const uint8_t *buffer, size_t length) override
Write a number of bytes.
Definition: strgen.cpp:304
Header of a language file.
Definition: language.h:24
char isocode[16]
the ISO code for the language (not country code)
Definition: language.h:31
char own_name[32]
the localized name of this language
Definition: language.h:30
char name[32]
the international name of this language
Definition: language.h:29
uint16_t winlangid
Windows language ID: Windows cannot and will not convert isocodes to something it can use to determin...
Definition: language.h:51
uint8_t num_genders
the number of genders of this language
Definition: language.h:53
char cases[MAX_NUM_CASES][CASE_GENDER_LEN]
the cases used by this translation
Definition: language.h:58
char digit_group_separator[8]
Thousand separator used for anything not currencies.
Definition: language.h:35
uint8_t newgrflangid
newgrf language id
Definition: language.h:52
uint8_t text_dir
default direction of the text
Definition: language.h:42
char genders[MAX_NUM_GENDERS][CASE_GENDER_LEN]
the genders used by this translation
Definition: language.h:57
uint8_t num_cases
the number of cases of this language
Definition: language.h:54
char digit_decimal_separator[8]
Decimal separator.
Definition: language.h:39
char digit_group_separator_currency[8]
Thousand separator used for currencies.
Definition: language.h:37
Base class for all language writers.
Definition: strgen.h:108
Data of an option.
Definition: getoptdata.h:21
OptionDataType type
The type of option.
Definition: getoptdata.h:22
Information about the currently known strings.
Definition: strgen.h:41
size_t next_string_id
The next string ID to allocate.
Definition: strgen.h:46
uint Version() const
Make a hash of the file to get a unique "version number".
Helper for reading strings.
Definition: strgen.h:58
const std::string file
The file we are reading.
Definition: strgen.h:60
StringData & data
The data to fill during reading.
Definition: strgen.h:59
virtual void ParseFile()
Start parsing the file.
bool translation
Are we reading a translation, implies !master. However, the base translation will have this false.
Definition: strgen.h:62
virtual void HandlePragma(char *str)
Handle the pragma of the file.
bool master
Are we reading the master file?
Definition: strgen.h:61