16 #include "string_base.h"
24 # define strncasecmp strnicmp
37 # include <unicode/ustring.h>
42 #if defined(WITH_COCOA)
60 void strecpy(std::span<char> dst, std::string_view src)
63 if (std::empty(dst) || std::size(src) >= std::size(dst) - 1U) {
64 #if defined(STRGEN) || defined(SETTINGSGEN)
65 FatalError(
"String too long for destination buffer");
67 Debug(misc, 0,
"String too long for destination buffer");
68 src = src.substr(0, std::size(dst) - 1U);
72 auto it = std::copy(std::begin(src), std::end(src), std::begin(dst));
84 str.reserve(data.size() * 2 + 1);
87 fmt::format_to(std::back_inserter(str),
"{:02X}", b);
111 while (str <= last && *str !=
'\0') {
137 if (len == 0 || str + len > last + 1 || len !=
Utf8Decode(&c, str)) {
149 }
while (--len != 0);
207 if (str.empty())
return {};
209 auto buf = str.data();
210 auto last = buf + str.size() - 1;
212 std::ostringstream dst;
213 std::ostreambuf_iterator<char> dst_iter(dst);
230 auto it = std::begin(str);
231 auto last = std::prev(std::end(str));
233 while (it <= last && *it !=
'\0') {
239 if (len == 0 || it + len > last)
return false;
243 if (!IsPrintable(c) || (c >= SCC_SPRITE_START && c <= SCC_SPRITE_END)) {
262 str = StrTrimView(str);
265 std::string_view StrTrimView(std::string_view str)
267 size_t first_pos = str.find_first_not_of(
' ');
268 if (first_pos == std::string::npos) {
269 return std::string_view{};
271 size_t last_pos = str.find_last_not_of(
' ');
272 return str.substr(first_pos, last_pos - first_pos + 1);
283 if (str.size() < prefix.size())
return false;
289 static bool eq(
char c1,
char c2) {
return toupper(c1) == toupper(c2); }
290 static bool ne(
char c1,
char c2) {
return toupper(c1) != toupper(c2); }
291 static bool lt(
char c1,
char c2) {
return toupper(c1) < toupper(c2); }
293 static int compare(
const char *s1,
const char *s2,
size_t n)
296 if (toupper(*s1) < toupper(*s2))
return -1;
297 if (toupper(*s1) > toupper(*s2))
return 1;
303 static const char *find(
const char *s,
size_t n,
char a)
305 for (; n > 0; --n, ++s) {
306 if (toupper(*s) == toupper(a))
return s;
323 if (str.size() < suffix.size())
return false;
338 return ci_str1.compare(ci_str2);
349 if (str1.size() != str2.size())
return false;
363 while (Utf8Consume(&t) != 0) len++;
378 bool strtolower(std::string &str, std::string::size_type offs)
380 bool changed =
false;
381 for (
auto ch = str.begin() + offs; ch != str.end(); ++ch) {
382 auto new_ch =
static_cast<char>(tolower(
static_cast<unsigned char>(*ch)));
383 changed |= new_ch != *ch;
400 case CS_NUMERAL:
return (key >=
'0' && key <=
'9');
403 case CS_ALPHA:
return IsPrintable(key) && !(key >=
'0' && key <=
'9');
404 case CS_HEXADECIMAL:
return (key >=
'0' && key <=
'9') || (key >=
'a' && key <=
'f') || (key >=
'A' && key <=
'F');
405 default: NOT_REACHED();
421 assert(c !=
nullptr);
427 }
else if (
GB(s[0], 5, 3) == 6) {
428 if (IsUtf8Part(s[1])) {
430 *c =
GB(s[0], 0, 5) << 6 |
GB(s[1], 0, 6);
431 if (*c >= 0x80)
return 2;
433 }
else if (
GB(s[0], 4, 4) == 14) {
434 if (IsUtf8Part(s[1]) && IsUtf8Part(s[2])) {
436 *c =
GB(s[0], 0, 4) << 12 |
GB(s[1], 0, 6) << 6 |
GB(s[2], 0, 6);
437 if (*c >= 0x800)
return 3;
439 }
else if (
GB(s[0], 3, 5) == 30) {
440 if (IsUtf8Part(s[1]) && IsUtf8Part(s[2]) && IsUtf8Part(s[3])) {
442 *c =
GB(s[0], 0, 3) << 18 |
GB(s[1], 0, 6) << 12 |
GB(s[2], 0, 6) << 6 |
GB(s[3], 0, 6);
443 if (*c >= 0x10000 && *c <= 0x10FFFF)
return 4;
465 }
else if (c < 0x800) {
466 *buf++ = 0xC0 +
GB(c, 6, 5);
467 *buf = 0x80 +
GB(c, 0, 6);
469 }
else if (c < 0x10000) {
470 *buf++ = 0xE0 +
GB(c, 12, 4);
471 *buf++ = 0x80 +
GB(c, 6, 6);
472 *buf = 0x80 +
GB(c, 0, 6);
474 }
else if (c < 0x110000) {
475 *buf++ = 0xF0 +
GB(c, 18, 3);
476 *buf++ = 0x80 +
GB(c, 12, 6);
477 *buf++ = 0x80 +
GB(c, 6, 6);
478 *buf = 0x80 +
GB(c, 0, 6);
488 return Utf8Encode<char *>(buf, c);
491 size_t Utf8Encode(std::ostreambuf_iterator<char> &buf, char32_t c)
493 return Utf8Encode<std::ostreambuf_iterator<char> &>(buf, c);
496 size_t Utf8Encode(std::back_insert_iterator<std::string> &buf, char32_t c)
498 return Utf8Encode<std::back_insert_iterator<std::string> &>(buf, c);
512 for (
const char *ptr = strchr(s,
'\0'); *s !=
'\0';) {
515 if (len == 0) len = 1;
519 if (length + len >= maxlen || (s + len > ptr))
break;
528 #ifdef DEFINE_STRCASESTR
529 char *strcasestr(
const char *haystack,
const char *needle)
531 size_t hay_len = strlen(haystack);
532 size_t needle_len = strlen(needle);
533 while (hay_len >= needle_len) {
534 if (strncasecmp(haystack, needle, needle_len) == 0)
return const_cast<char *
>(haystack);
551 if (c >=
'0' && c <=
'9')
return false;
552 if (c >=
'A' && c <=
'Z')
return false;
553 if (c >=
'a' && c <=
'z')
return false;
554 if (c >= SCC_CONTROL_START && c <= SCC_CONTROL_END)
return true;
555 if (c >= 0xC0 && c <= 0x10FFFF)
return false;
570 auto first = std::begin(str);
571 auto last = std::end(str);
572 while (first < last) {
578 return {first, last};
591 if (ignore_garbage_at_front) {
598 UErrorCode status = U_ZERO_ERROR;
599 int result =
_current_collator->compareUTF8(icu::StringPiece(s1.data(), s1.size()), icu::StringPiece(s2.data(), s2.size()), status);
600 if (U_SUCCESS(status))
return result;
604 #if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
605 int res = OTTDStringCompare(s1, s2);
606 if (res != 0)
return res - 2;
609 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
611 if (res != 0)
return res - 2;
620 #include <unicode/stsearch.h>
630 static int ICUStringContains(
const std::string_view str,
const std::string_view value,
bool case_insensitive)
633 std::unique_ptr<icu::RuleBasedCollator> coll(
dynamic_cast<icu::RuleBasedCollator *
>(
_current_collator->clone()));
635 UErrorCode status = U_ZERO_ERROR;
636 coll->setStrength(case_insensitive ? icu::Collator::SECONDARY : icu::Collator::TERTIARY);
637 coll->setAttribute(UCOL_NUMERIC_COLLATION, UCOL_OFF, status);
639 auto u_str = icu::UnicodeString::fromUTF8(icu::StringPiece(str.data(), str.size()));
640 auto u_value = icu::UnicodeString::fromUTF8(icu::StringPiece(value.data(), value.size()));
641 icu::StringSearch u_searcher(u_value, u_str, coll.get(),
nullptr, status);
642 if (U_SUCCESS(status)) {
643 auto pos = u_searcher.first(status);
644 if (U_SUCCESS(status))
return pos != USEARCH_DONE ? 1 : 0;
664 if (res_u >= 0)
return res_u > 0;
667 #if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
669 if (res >= 0)
return res > 0;
672 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
674 if (res >= 0)
return res > 0;
677 return str.find(value) != std::string_view::npos;
691 if (res_u >= 0)
return res_u > 0;
694 #if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
696 if (res >= 0)
return res > 0;
699 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
701 if (res >= 0)
return res > 0;
706 return ci_str.find(ci_value) != CaseInsensitiveStringView::npos;
717 if (c >=
'0' && c <=
'9')
return c -
'0';
718 if (c >=
'A' && c <=
'F')
return c + 10 -
'A';
719 if (c >=
'a' && c <=
'f')
return c + 10 -
'a';
736 if (bytes.size() != hex.size() / 2) {
741 if (hex.size() % 2 != 0) {
745 for (
size_t i = 0; i < hex.size() / 2; i++) {
749 if (hi < 0 || lo < 0) {
753 bytes[i] = (hi << 4) | lo;
759 #ifdef WITH_UNISCRIBE
763 return std::make_unique<UniscribeStringIterator>();
766 #elif defined(WITH_ICU_I18N)
768 #include <unicode/utext.h>
769 #include <unicode/brkiter.h>
783 UErrorCode status = U_ZERO_ERROR;
787 this->utf16_str.push_back(
'\0');
788 this->utf16_to_utf8.push_back(0);
799 const char *string_base = s;
805 this->utf16_str.clear();
806 this->utf16_to_utf8.clear();
809 size_t idx = s - string_base;
811 char32_t c = Utf8Consume(&s);
813 this->utf16_str.push_back((UChar)c);
816 this->utf16_str.push_back((UChar)(0xD800 + ((c - 0x10000) >> 10)));
817 this->utf16_str.push_back((UChar)(0xDC00 + ((c - 0x10000) & 0x3FF)));
818 this->utf16_to_utf8.push_back(idx);
820 this->utf16_to_utf8.push_back(idx);
822 this->utf16_str.push_back(
'\0');
823 this->utf16_to_utf8.push_back(s - string_base);
825 UText text = UTEXT_INITIALIZER;
826 UErrorCode status = U_ZERO_ERROR;
827 utext_openUChars(&text, this->utf16_str.data(), this->utf16_str.size() - 1, &status);
828 this->char_itr->setText(&text, status);
829 this->word_itr->setText(&text, status);
830 this->char_itr->first();
831 this->word_itr->first();
838 for (uint i = 0; i < this->utf16_to_utf8.size(); i++) {
839 if (this->utf16_to_utf8[i] == pos) {
848 this->char_itr->isBoundary(utf16_pos);
849 return this->utf16_to_utf8[this->char_itr->current()];
857 pos = this->char_itr->next();
861 pos = this->word_itr->following(this->char_itr->current());
865 while (pos != icu::BreakIterator::DONE &&
867 int32_t new_pos = this->word_itr->next();
870 if (new_pos == icu::BreakIterator::DONE)
break;
874 this->char_itr->isBoundary(pos);
881 return pos == icu::BreakIterator::DONE ?
END : this->utf16_to_utf8[pos];
889 pos = this->char_itr->previous();
893 pos = this->word_itr->preceding(this->char_itr->current());
897 while (pos != icu::BreakIterator::DONE &&
899 int32_t new_pos = this->word_itr->previous();
902 if (new_pos == icu::BreakIterator::DONE)
break;
906 this->char_itr->isBoundary(pos);
913 return pos == icu::BreakIterator::DONE ?
END : this->utf16_to_utf8[pos];
919 return std::make_unique<IcuStringIterator>();
932 DefaultStringIterator() : string(nullptr), len(0), cur_pos(0)
939 this->len = strlen(s);
945 assert(this->
string !=
nullptr && pos <= this->len);
947 while (pos > 0 && IsUtf8Part(this->
string[pos])) pos--;
948 return this->cur_pos = pos;
951 size_t Next(IterType what)
override
953 assert(this->
string !=
nullptr);
956 if (this->cur_pos >= this->len)
return END;
961 this->cur_pos +=
Utf8Decode(&c, this->
string + this->cur_pos);
962 return this->cur_pos;
968 size_t offs =
Utf8Decode(&c, this->
string + this->cur_pos);
970 this->cur_pos += offs;
971 offs =
Utf8Decode(&c, this->
string + this->cur_pos);
975 this->cur_pos += offs;
976 offs =
Utf8Decode(&c, this->
string + this->cur_pos);
979 return this->cur_pos;
989 size_t Prev(IterType what)
override
991 assert(this->
string !=
nullptr);
994 if (this->cur_pos == 0)
return END;
998 return this->cur_pos =
Utf8PrevChar(this->
string + this->cur_pos) - this->string;
1001 const char *s = this->
string + this->cur_pos;
1016 return this->cur_pos = s - this->string;
1027 #if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
1030 std::unique_ptr<StringIterator> i = OSXStringIterator::Create();
1031 if (i !=
nullptr)
return i;
1033 return std::make_unique<DefaultStringIterator>();
1038 return std::make_unique<DefaultStringIterator>();
Functions related to the allocation of memory.
constexpr debug_inline bool HasBit(const T x, const uint8_t y)
Checks if a bit in a value is set.
constexpr static debug_inline uint GB(const T x, const uint8_t s, const uint8_t n)
Fetch n bits from x, started at bit s.
String iterator using ICU as a backend.
void SetString(const char *s) override
Set a new iteration string.
size_t Prev(IterType what) override
Move the cursor back by one iteration unit.
size_t Next(IterType what) override
Advance the cursor by one iteration unit.
std::vector< size_t > utf16_to_utf8
Mapping from UTF-16 code point position to index in the UTF-8 source string.
size_t SetCurPosition(size_t pos) override
Change the current string cursor.
std::vector< UChar > utf16_str
UTF-16 copy of the string.
icu::BreakIterator * char_itr
ICU iterator for characters.
icu::BreakIterator * word_itr
ICU iterator for words.
Class for iterating over different kind of parts of a string.
static const size_t END
Sentinel to indicate end-of-iteration.
virtual size_t Prev(IterType what=ITER_CHARACTER)=0
Move the cursor back by one iteration unit.
virtual size_t SetCurPosition(size_t pos)=0
Change the current string cursor.
virtual size_t Next(IterType what=ITER_CHARACTER)=0
Advance the cursor by one iteration unit.
IterType
Type of the iterator.
@ ITER_WORD
Iterate over words.
@ ITER_CHARACTER
Iterate over characters (or more exactly grapheme clusters).
static std::unique_ptr< StringIterator > Create()
Create a new iterator instance.
virtual void SetString(const char *s)=0
Set a new iteration string.
Control codes that are embedded in the translation strings.
Functions related to debugging.
#define Debug(category, level, format_string,...)
Ouptut a line of debugging information.
Error reporting related functions.
fluid_settings_t * settings
FluidSynth settings handle.
Functions related to the gfx engine.
Information about languages and their files.
const LanguageMetadata * _current_language
The currently loaded language.
std::unique_ptr< icu::Collator > _current_collator
Collator for the language currently in use.
A number of safeguards to prevent using unsafe methods.
Definition of base types and functions in a cross-platform compatible way.
static void StrMakeValid(T &dst, const char *str, const char *last, StringValidationSettings settings)
Copies the valid (UTF-8) characters from str up to last to the dst.
bool ConvertHexToBytes(std::string_view hex, std::span< uint8_t > bytes)
Convert a hex-string to a byte-array, while validating it was actually hex.
bool IsValidChar(char32_t key, CharSetFilter afilter)
Only allow certain keys.
bool StrEqualsIgnoreCase(const std::string_view str1, const std::string_view str2)
Compares two string( view)s for equality, while ignoring the case of the characters.
bool StrNaturalContains(const std::string_view str, const std::string_view value)
Checks if a string is contained in another string with a locale-aware comparison that is case sensiti...
void StrMakeValidInPlace(char *str, const char *last, StringValidationSettings settings)
Scans the string for invalid characters and replaces then with a question mark '?' (if not ignored).
void strecpy(std::span< char > dst, std::string_view src)
Copies characters from one buffer to another.
std::string FormatArrayAsHex(std::span< const uint8_t > data)
Format a byte array into a continuous hex string.
size_t Utf8StringLength(const char *s)
Get the length of an UTF-8 encoded string in number of characters and thus not the number of bytes th...
bool StrStartsWithIgnoreCase(std::string_view str, const std::string_view prefix)
Check whether the given string starts with the given prefix, ignoring case.
bool StrValid(std::span< const char > str)
Checks whether the given string is valid, i.e.
static int ConvertHexNibbleToByte(char c)
Convert a single hex-nibble to a byte.
static int ICUStringContains(const std::string_view str, const std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
static std::string_view SkipGarbage(std::string_view str)
Skip some of the 'garbage' in the string that we don't want to use to sort on.
size_t Utf8Decode(char32_t *c, const char *s)
Decode and consume the next UTF-8 encoded character.
int StrNaturalCompare(std::string_view s1, std::string_view s2, bool ignore_garbage_at_front)
Compares two strings using case insensitive natural sort.
bool StrNaturalContainsIgnoreCase(const std::string_view str, const std::string_view value)
Checks if a string is contained in another string with a locale-aware comparison that is case insensi...
std::basic_string_view< char, CaseInsensitiveCharTraits > CaseInsensitiveStringView
Case insensitive string view.
int StrCompareIgnoreCase(const std::string_view str1, const std::string_view str2)
Compares two string( view)s, while ignoring the case of the characters.
bool StrEndsWithIgnoreCase(std::string_view str, const std::string_view suffix)
Check whether the given string ends with the given suffix, ignoring case.
size_t Utf8TrimString(char *s, size_t maxlen)
Properly terminate an UTF8 string to some maximum length.
void StrTrimInPlace(std::string &str)
Trim the spaces from given string in place, i.e.
size_t Utf8Encode(T buf, char32_t c)
Encode a unicode character and place it in the buffer.
static bool IsGarbageCharacter(char32_t c)
Test if a unicode character is considered garbage to be skipped.
Functions related to low-level strings.
char32_t Utf16DecodeChar(const uint16_t *c)
Decode an UTF-16 character.
bool IsWhitespace(char32_t c)
Check whether UNICODE character is whitespace or not, i.e.
int8_t Utf8EncodedCharLen(char c)
Return the length of an UTF-8 encoded value based on a single char.
char * Utf8PrevChar(char *s)
Retrieve the previous UNICODE character in an UTF-8 encoded string.
int MacOSStringCompare(std::string_view s1, std::string_view s2)
Compares two strings using case insensitive natural sort.
int MacOSStringContains(const std::string_view str, const std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
Functions related to localized text support on OSX.
CharSetFilter
Valid filter types for IsValidChar.
@ CS_NUMERAL_SPACE
Only numbers and spaces.
@ CS_HEXADECIMAL
Only hexadecimal characters.
@ CS_NUMERAL
Only numeric ones.
@ CS_NUMERAL_SIGNED
Only numbers and '-' for negative values.
@ CS_ALPHA
Only alphabetic values.
@ CS_ALPHANUMERAL
Both numeric and alphabetic and spaces and stuff.
StringValidationSettings
Settings for the string validation.
@ SVS_ALLOW_CONTROL_CODE
Allow the special control codes.
@ SVS_REPLACE_TAB_CR_NL_WITH_SPACE
Replace tabs ('\t'), carriage returns ('\r') and newlines (' ') with spaces.
@ SVS_ALLOW_NEWLINE
Allow newlines; replaces '\r ' with ' ' during processing.
@ SVS_REPLACE_WITH_QUESTION_MARK
Replace the unknown/bad bits with question marks.
Functions related to laying out text on Win32.
Case insensitive implementation of the standard character type traits.
int Win32StringContains(const std::string_view str, const std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
declarations of functions for MS windows systems