10 #include "../stdafx.h"
11 #include "../core/alloc_func.hpp"
12 #include "../core/endian_func.hpp"
13 #include "../core/mem_func.hpp"
14 #include "../error_func.h"
15 #include "../string_func.h"
16 #include "../table/control_codes.h"
21 #include "../table/strgen_tables.h"
23 #include "../safeguards.h"
29 const char *
_file =
"(unknown file)";
31 int _errors, _warnings, _show_todo;
35 static const CmdStruct *ParseCommandString(
const char **str,
char *param,
int *argno,
int *casei);
43 caseidx(caseidx), string(string)
55 name(name), english(english), index(index), line(line)
93 this->
strings[ls->index].swap(ls);
117 for (; *s !=
'\0'; s++) {
118 hash = std::rotl(hash, 3) ^ *s;
119 hash = (hash & 1 ? hash >> 1 ^ 0xDEADBEEF : hash >> 1);
142 s = ls->
name.c_str();
143 hash ^= i * 0x717239;
144 hash = (hash & 1 ? hash >> 1 ^ 0xDEADBEEF : hash >> 1);
148 while ((cs = ParseCommandString(&s, buf, &argno, &casei)) !=
nullptr) {
151 hash ^= (cs - _cmd_structs) * 0x1234567;
152 hash = (hash & 1 ? hash >> 1 ^ 0xF00BAA4 : hash >> 1);
171 static const char *_cur_ident;
175 static int _cur_argidx;
185 this->push_back(value);
195 this->push_back(value);
196 }
else if (value < 0x800) {
197 this->push_back(0xC0 +
GB(value, 6, 5));
198 this->push_back(0x80 +
GB(value, 0, 6));
199 }
else if (value < 0x10000) {
200 this->push_back(0xE0 +
GB(value, 12, 4));
201 this->push_back(0x80 +
GB(value, 6, 6));
202 this->push_back(0x80 +
GB(value, 0, 6));
203 }
else if (value < 0x110000) {
204 this->push_back(0xF0 +
GB(value, 18, 3));
205 this->push_back(0x80 +
GB(value, 12, 6));
206 this->push_back(0x80 +
GB(value, 6, 6));
207 this->push_back(0x80 +
GB(value, 0, 6));
209 StrgenWarning(
"Invalid unicode value U+0x{:X}", value);
214 size_t Utf8Validate(
const char *s)
221 }
else if (
GB(s[0], 5, 3) == 6 && IsUtf8Part(s[1])) {
223 c =
GB(s[0], 0, 5) << 6 |
GB(s[1], 0, 6);
224 if (c >= 0x80)
return 2;
225 }
else if (
GB(s[0], 4, 4) == 14 && IsUtf8Part(s[1]) && IsUtf8Part(s[2])) {
227 c =
GB(s[0], 0, 4) << 12 |
GB(s[1], 0, 6) << 6 |
GB(s[2], 0, 6);
228 if (c >= 0x800)
return 3;
229 }
else if (
GB(s[0], 3, 5) == 30 && IsUtf8Part(s[1]) && IsUtf8Part(s[2]) && IsUtf8Part(s[3])) {
231 c =
GB(s[0], 0, 3) << 18 |
GB(s[1], 0, 6) << 12 |
GB(s[2], 0, 6) << 6 |
GB(s[3], 0, 6);
232 if (c >= 0x10000 && c <= 0x10FFFF)
return 4;
239 void EmitSingleChar(
Buffer *buffer,
char *buf,
int value)
241 if (*buf !=
'\0') StrgenWarning(
"Ignoring trailing letters in command");
252 bool ParseRelNum(
char **buf,
int *value,
int *offset)
254 const char *s = *buf;
258 while (*s ==
' ' || *s ==
'\t') s++;
263 int v = std::strtol(s, &end, 0);
264 if (end == s)
return false;
270 if (offset !=
nullptr && *end ==
':') {
273 *offset = std::strtol(s, &end, 0);
274 if (end == s)
return false;
281 char *ParseWord(
char **buf)
285 while (*s ==
' ' || *s ==
'\t') s++;
286 if (*s ==
'\0')
return nullptr;
292 if (*s ==
'\0')
break;
303 if (*s ==
'\0')
break;
304 if (*s ==
' ' || *s ==
'\t') {
316 static int TranslateArgumentIdx(
int arg,
int offset = 0);
318 static void EmitWordList(
Buffer *buffer,
const std::vector<const char *> &words, uint nw)
321 for (uint i = 0; i < nw; i++) buffer->
AppendByte((uint8_t)strlen(words[i]) + 1);
322 for (uint i = 0; i < nw; i++) {
323 for (uint j = 0; words[i][j] !=
'\0'; j++) buffer->
AppendByte(words[i][j]);
328 void EmitPlural(
Buffer *buffer,
char *buf,
int)
330 int argidx = _cur_argidx;
333 std::vector<const char *> words(std::max(expected,
MAX_PLURALS),
nullptr);
337 if (!ParseRelNum(&buf, &argidx, &offset)) argidx--;
339 const CmdStruct *cmd = _cur_pcs.consuming_commands[argidx];
342 if (cmd ==
nullptr || cmd->default_plural_offset < 0) {
343 StrgenFatal(
"Command '{}' has no (default) plural position", cmd ==
nullptr ?
"<empty>" : cmd->cmd);
345 offset = cmd->default_plural_offset;
350 words[nw] = ParseWord(&buf);
351 if (words[nw] ==
nullptr)
break;
355 StrgenFatal(
"{}: No plural words", _cur_ident);
358 if (expected != nw) {
360 StrgenFatal(
"{}: Invalid number of plural forms. Expecting {}, found {}.", _cur_ident,
363 if ((_show_todo & 2) != 0) StrgenWarning(
"'{}' is untranslated. Tweaking english string to allow compilation for plural forms", _cur_ident);
367 for (; nw < expected; nw++) {
368 words[nw] = words[nw - 1];
376 buffer->
AppendByte(TranslateArgumentIdx(argidx, offset));
377 EmitWordList(buffer, words, nw);
380 void EmitGender(
Buffer *buffer,
char *buf,
int)
382 int argidx = _cur_argidx;
391 if (nw >=
MAX_NUM_GENDERS) StrgenFatal(
"G argument '{}' invalid", buf);
401 ParseRelNum(&buf, &argidx, &offset);
403 const CmdStruct *cmd = _cur_pcs.consuming_commands[argidx];
404 if (cmd ==
nullptr || (cmd->flags &
C_GENDER) == 0) {
405 StrgenFatal(
"Command '{}' can't have a gender", cmd ==
nullptr ?
"<empty>" : cmd->cmd);
409 words[nw] = ParseWord(&buf);
410 if (words[nw] ==
nullptr)
break;
412 if (nw !=
_lang.
num_genders) StrgenFatal(
"Bad # of arguments for gender command");
414 assert(
IsInsideBS(cmd->value, SCC_CONTROL_START, UINT8_MAX));
416 buffer->
AppendByte(TranslateArgumentIdx(argidx, offset));
417 EmitWordList(buffer, words, nw);
421 static const CmdStruct *FindCmd(
const char *s,
int len)
423 for (
const auto &cs : _cmd_structs) {
424 if (strncmp(cs.cmd, s, len) == 0 && cs.cmd[len] ==
'\0')
return &cs;
429 static uint ResolveCaseName(
const char *str,
size_t len)
433 len = std::min(
lengthof(case_str) - 1, len);
434 memcpy(case_str, str, len);
435 case_str[len] =
'\0';
438 if (case_idx >=
MAX_NUM_CASES) StrgenFatal(
"Invalid case-name '{}'", case_str);
445 static const CmdStruct *ParseCommandString(
const char **str,
char *param,
int *argno,
int *casei)
447 const char *s = *str, *start;
454 for (; *s !=
'{'; s++) {
455 if (*s ==
'\0')
return nullptr;
459 if (*s >=
'0' && *s <=
'9') {
462 *argno = std::strtoul(s, &end, 0);
463 if (*end !=
':') StrgenFatal(
"missing arg #");
471 }
while (c !=
'}' && c !=
' ' && c !=
'=' && c !=
'.' && c != 0);
473 const CmdStruct *cmd = FindCmd(start, s - start - 1);
474 if (cmd ==
nullptr) {
475 std::string command(start, s - start - 1);
476 StrgenError(
"Undefined command '{}'", command);
481 const char *casep = s;
483 if (!(cmd->flags &
C_CASE)) {
484 StrgenFatal(
"Command '{}' can't have a case", cmd->cmd);
489 }
while (c !=
'}' && c !=
' ' && c !=
'\0');
490 *casei = ResolveCaseName(casep, s - casep - 1);
494 StrgenError(
"Missing }} from command '{}'", start);
507 StrgenError(
"Missing }} from command '{}'", start);
529 data(data), file(file), master(master), translation(translation)
544 const CmdStruct *ar = ParseCommandString(&s, param, &argno, &casei);
546 if (ar ==
nullptr)
break;
549 if (argno != -1 && ar->consumes == 0) StrgenFatal(
"Non consumer param can't have a paramindex");
552 if (argno != -1) argidx = argno;
553 if (argidx < 0 || (uint)argidx >= p.consuming_commands.max_size()) StrgenFatal(
"invalid param idx {}", argidx);
554 if (p.consuming_commands[argidx] !=
nullptr && p.consuming_commands[argidx] != ar) StrgenFatal(
"duplicate param idx {}", argidx);
556 p.consuming_commands[argidx++] = ar;
558 p.non_consuming_commands.emplace_back(
CmdPair{ar, param});
568 if (a ==
nullptr)
return nullptr;
570 if (strcmp(a->cmd,
"STRING1") == 0 ||
571 strcmp(a->cmd,
"STRING2") == 0 ||
572 strcmp(a->cmd,
"STRING3") == 0 ||
573 strcmp(a->cmd,
"STRING4") == 0 ||
574 strcmp(a->cmd,
"STRING5") == 0 ||
575 strcmp(a->cmd,
"STRING6") == 0 ||
576 strcmp(a->cmd,
"STRING7") == 0 ||
577 strcmp(a->cmd,
"RAW_STRING") == 0) {
578 return FindCmd(
"STRING", 6);
585 static bool CheckCommandsMatch(
const char *a,
const char *b,
const char *name)
599 if (templ.non_consuming_commands.max_size() != lang.non_consuming_commands.max_size()) {
600 StrgenWarning(
"{}: template string and language string have a different # of commands", name);
604 for (
auto &templ_nc : templ.non_consuming_commands) {
607 for (
auto &lang_nc : lang.non_consuming_commands) {
608 if (templ_nc.cmd == lang_nc.cmd && templ_nc.param == lang_nc.param) {
610 lang_nc.cmd =
nullptr;
617 StrgenWarning(
"{}: command '{}' exists in template file but not in language file", name, templ_nc.cmd->cmd);
624 for (uint i = 0; i < templ.consuming_commands.max_size(); i++) {
625 if (TranslateCmdForCompare(templ.consuming_commands[i]) != lang.consuming_commands[i]) {
626 StrgenWarning(
"{}: Param idx #{} '{}' doesn't match with template command '{}'", name, i,
627 lang.consuming_commands[i] ==
nullptr ?
"<empty>" : TranslateCmdForCompare(lang.consuming_commands[i])->cmd,
628 templ.consuming_commands[i] ==
nullptr ?
"<empty>" : templ.consuming_commands[i]->cmd);
636 void StringReader::HandleString(
char *str)
639 if (str[1] ==
'#' && str[2] !=
'#') this->
HandlePragma(str + 2);
644 if (*str ==
';' || *str ==
' ' || *str ==
'\0')
return;
646 char *s = strchr(str,
':');
648 StrgenError(
"Line has no ':' delimiter");
655 for (t = s; t > str && (t[-1] ==
' ' || t[-1] ==
'\t'); t--) {}
661 for (tmp = s; *tmp !=
'\0';) {
662 size_t len = Utf8Validate(tmp);
663 if (len == 0) StrgenFatal(
"Invalid UTF-8 sequence in '{}'", s);
669 (c >= 0xE000 && c <= 0xF8FF) ||
670 (c >= 0xFFF0 && c <= 0xFFFF)) {
671 StrgenFatal(
"Unwanted UTF-8 character U+{:04X} in sequence '{}'", (
int)c, s);
679 char *casep = strchr(str,
'.');
680 if (casep !=
nullptr) *casep++ =
'\0';
686 if (casep !=
nullptr) {
687 StrgenError(
"Cases in the base translation are not supported.");
691 if (ent !=
nullptr) {
692 StrgenError(
"String name '{}' is used multiple times", str);
696 if (this->
data.
strings[this->data.next_string_id] !=
nullptr) {
697 StrgenError(
"String ID 0x{:X} for '{}' already in use by '{}'", this->
data.
next_string_id, str, this->data.strings[this->data.next_string_id]->name);
704 if (ent ==
nullptr) {
705 StrgenWarning(
"String name '{}' does not exist in master file", str);
709 if (!ent->
translated.empty() && casep ==
nullptr) {
710 StrgenError(
"String name '{}' is used multiple times", str);
715 if (!CheckCommandsMatch(s, ent->
english.c_str(), str))
return;
717 if (casep !=
nullptr) {
718 ent->
translated_cases.emplace_back(ResolveCaseName(casep, strlen(casep)), s);
731 if (!memcmp(str,
"plural ", 7)) {
737 StrgenFatal(
"unknown pragma '{}'", str);
741 static void StripTrailingWhitespace(std::string &str)
743 str.erase(str.find_last_not_of(
"\r\n ") + 1);
748 _warnings = _errors = 0;
765 std::optional<std::string> line = this->
ReadLine();
766 if (!line.has_value())
return;
768 StripTrailingWhitespace(line.value());
769 this->HandleString(line.value().data());
774 StrgenError(
"Too many strings, maximum allowed is {}", this->
data.
max_strings);
786 if (data.
strings[i] !=
nullptr) {
795 static int TranslateArgumentIdx(
int argidx,
int offset)
799 if (argidx < 0 || (uint)argidx >= _cur_pcs.consuming_commands.max_size()) {
800 StrgenFatal(
"invalid argidx {}", argidx);
802 const CmdStruct *cs = _cur_pcs.consuming_commands[argidx];
803 if (cs !=
nullptr && cs->consumes <= offset) {
804 StrgenFatal(
"invalid argidx offset {}:{}", argidx, offset);
807 if (_cur_pcs.consuming_commands[argidx] ==
nullptr) {
808 StrgenFatal(
"no command for this argidx {}", argidx);
811 for (
int i = sum = 0; i < argidx; i++) {
812 cs = _cur_pcs.consuming_commands[i];
814 sum += (cs !=
nullptr) ? cs->consumes : 1;
820 static void PutArgidxCommand(
Buffer *buffer)
823 buffer->
AppendByte(TranslateArgumentIdx(_cur_argidx));
827 static void PutCommandString(
Buffer *buffer,
const char *str)
831 while (*str !=
'\0') {
841 const CmdStruct *cs = ParseCommandString(&str, param, &argno, &casei);
842 if (cs ==
nullptr)
break;
850 if (cs->consumes > 0) {
852 if (argno != -1 && argno != _cur_argidx) {
854 PutArgidxCommand(buffer);
858 cs = _cur_pcs.consuming_commands[_cur_argidx++];
860 StrgenFatal(
"{}: No argument exists at position {}", _cur_ident, _cur_argidx - 1);
864 cs->proc(buffer, param, cs->value);
876 if (length >= 0x4000) {
877 StrgenFatal(
"string too long");
880 if (length >= 0xC0) {
881 buffer[offs++] = (length >> 8) | 0xC0;
883 buffer[offs++] = length & 0xFF;
884 this->
Write((uint8_t*)buffer, offs);
893 std::vector<uint> in_use;
894 for (
size_t tab = 0; tab < data.
tabs; tab++) {
900 for (uint j = 0; j != in_use[tab]; j++) {
914 for (
size_t tab = 0; tab < data.
tabs; tab++) {
915 for (uint j = 0; j != in_use[tab]; j++) {
917 const std::string *cmdp;
925 _cur_ident = ls->
name.c_str();
929 if (_show_todo > 0 && ls->
translated.empty()) {
930 if ((_show_todo & 2) != 0) {
931 StrgenWarning(
"'{}' is untranslated", ls->
name);
933 if ((_show_todo & 1) != 0) {
934 const char *s =
"<TODO> ";
940 _cur_pcs = ExtractCommandString(ls->
english.c_str(),
false);
962 uint pos = (uint)buffer.size();
966 PutCommandString(&buffer, c.
string.c_str());
969 uint size = (uint)buffer.size() - (pos + 2);
970 buffer[pos + 0] =
GB(size, 8, 8);
971 buffer[pos + 1] =
GB(size, 0, 8);
975 if (!cmdp->empty()) PutCommandString(&buffer, cmdp->c_str());
978 this->
Write(buffer.data(), buffer.size());