OpenTTD Source 20250529-master-g10c159a79f
string.cpp
Go to the documentation of this file.
1/*
2 * This file is part of OpenTTD.
3 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
6 */
7
10#include "stdafx.h"
11#include "debug.h"
12#include "core/math_func.hpp"
13#include "error_func.h"
14#include "string_func.h"
15#include "string_base.h"
16#include "core/utf8.hpp"
18
19#include "table/control_codes.h"
20
21#ifdef _WIN32
22# include "os/windows/win32.h"
23#endif
24
25#ifdef WITH_UNISCRIBE
27#endif
28
29#ifdef WITH_ICU_I18N
30/* Required by StrNaturalCompare. */
31# include <unicode/ustring.h>
32# include "language.h"
33# include "gfx_func.h"
34#endif /* WITH_ICU_I18N */
35
36#if defined(WITH_COCOA)
37# include "os/macosx/string_osx.h"
38#endif
39
40#include "safeguards.h"
41
42
54void strecpy(std::span<char> dst, std::string_view src)
55{
56 /* Ensure source string fits with NUL terminator; dst must be at least 1 character longer than src. */
57 if (std::empty(dst) || std::size(src) >= std::size(dst) - 1U) {
58#if defined(STRGEN) || defined(SETTINGSGEN)
59 FatalError("String too long for destination buffer");
60#else /* STRGEN || SETTINGSGEN */
61 Debug(misc, 0, "String too long for destination buffer");
62 src = src.substr(0, std::size(dst) - 1U);
63#endif /* STRGEN || SETTINGSGEN */
64 }
65
66 auto it = std::copy(std::begin(src), std::end(src), std::begin(dst));
67 *it = '\0';
68}
69
75std::string FormatArrayAsHex(std::span<const uint8_t> data)
76{
77 std::string str;
78 str.reserve(data.size() * 2 + 1);
79
80 for (auto b : data) {
81 format_append(str, "{:02X}", b);
82 }
83
84 return str;
85}
86
92static bool IsSccEncodedCode(char32_t c)
93{
94 switch (c) {
95 case SCC_RECORD_SEPARATOR:
96 case SCC_ENCODED:
100 return true;
101
102 default:
103 return false;
104 }
105}
106
116template <class Builder>
117static void StrMakeValid(Builder &builder, StringConsumer &consumer, StringValidationSettings settings)
118{
119 /* Assume the ABSOLUTE WORST to be in str as it comes from the outside. */
120 while (consumer.AnyBytesLeft()) {
121 auto c = consumer.TryReadUtf8();
122 if (!c.has_value()) {
123 /* Maybe the next byte is still a valid character? */
124 consumer.Skip(1);
125 continue;
126 }
127 if (*c == 0) break;
128
129 if ((IsPrintable(*c) && (*c < SCC_SPRITE_START || *c > SCC_SPRITE_END)) ||
131 (settings.Test(StringValidationSetting::AllowNewline) && *c == '\n')) {
132 builder.PutUtf8(*c);
133 } else if (settings.Test(StringValidationSetting::AllowNewline) && *c == '\r' && consumer.PeekCharIf('\n')) {
134 /* Skip \r, if followed by \n */
135 /* continue */
136 } else if (settings.Test(StringValidationSetting::ReplaceTabCrNlWithSpace) && (*c == '\r' || *c == '\n' || *c == '\t')) {
137 /* Replace the tab, carriage return or newline with a space. */
138 builder.PutChar(' ');
140 /* Replace the undesirable character with a question mark */
141 builder.PutChar('?');
142 }
143 }
144
145 /* String termination, if needed, is left to the caller of this function. */
146}
147
156{
157 InPlaceReplacement inplace(std::span(str, strlen(str)));
158 StrMakeValid(inplace.builder, inplace.consumer, settings);
159 /* Add NUL terminator, if we ended up with less bytes than before */
160 if (inplace.builder.AnyBytesUnused()) inplace.builder.PutChar('\0');
161}
162
171{
172 if (str.empty()) return;
173
174 InPlaceReplacement inplace(std::span(str.data(), str.size()));
175 StrMakeValid(inplace.builder, inplace.consumer, settings);
176 str.erase(inplace.builder.GetBytesWritten(), std::string::npos);
177}
178
186std::string StrMakeValid(std::string_view str, StringValidationSettings settings)
187{
188 std::string result;
189 StringBuilder builder(result);
190 StringConsumer consumer(str);
191 StrMakeValid(builder, consumer, settings);
192 return result;
193}
194
203bool StrValid(std::span<const char> str)
204{
205 /* Assume the ABSOLUTE WORST to be in str as it comes from the outside. */
206 StringConsumer consumer(str);
207 while (consumer.AnyBytesLeft()) {
208 auto c = consumer.TryReadUtf8();
209 if (!c.has_value()) return false; // invalid codepoint
210 if (*c == 0) return true; // NUL termination
211 if (!IsPrintable(*c) || (*c >= SCC_SPRITE_START && *c <= SCC_SPRITE_END)) {
212 return false;
213 }
214 }
215
216 return false; // missing NUL termination
217}
218
226void StrTrimInPlace(std::string &str)
227{
228 size_t first_pos = str.find_first_not_of(StringConsumer::WHITESPACE_NO_NEWLINE);
229 if (first_pos == std::string::npos) {
230 str.clear();
231 return;
232 }
233 str.erase(0, first_pos);
234
235 size_t last_pos = str.find_last_not_of(StringConsumer::WHITESPACE_NO_NEWLINE);
236 str.erase(last_pos + 1);
237}
238
239std::string_view StrTrimView(std::string_view str, std::string_view characters_to_trim)
240{
241 size_t first_pos = str.find_first_not_of(characters_to_trim);
242 if (first_pos == std::string::npos) {
243 return std::string_view{};
244 }
245 size_t last_pos = str.find_last_not_of(characters_to_trim);
246 return str.substr(first_pos, last_pos - first_pos + 1);
247}
248
255bool StrStartsWithIgnoreCase(std::string_view str, std::string_view prefix)
256{
257 if (str.size() < prefix.size()) return false;
258 return StrEqualsIgnoreCase(str.substr(0, prefix.size()), prefix);
259}
260
262struct CaseInsensitiveCharTraits : public std::char_traits<char> {
263 static bool eq(char c1, char c2) { return toupper(c1) == toupper(c2); }
264 static bool ne(char c1, char c2) { return toupper(c1) != toupper(c2); }
265 static bool lt(char c1, char c2) { return toupper(c1) < toupper(c2); }
266
267 static int compare(const char *s1, const char *s2, size_t n)
268 {
269 while (n-- != 0) {
270 if (toupper(*s1) < toupper(*s2)) return -1;
271 if (toupper(*s1) > toupper(*s2)) return 1;
272 ++s1; ++s2;
273 }
274 return 0;
275 }
276
277 static const char *find(const char *s, size_t n, char a)
278 {
279 for (; n > 0; --n, ++s) {
280 if (toupper(*s) == toupper(a)) return s;
281 }
282 return nullptr;
283 }
284};
285
287typedef std::basic_string_view<char, CaseInsensitiveCharTraits> CaseInsensitiveStringView;
288
295bool StrEndsWithIgnoreCase(std::string_view str, std::string_view suffix)
296{
297 if (str.size() < suffix.size()) return false;
298 return StrEqualsIgnoreCase(str.substr(str.size() - suffix.size()), suffix);
299}
300
308int StrCompareIgnoreCase(std::string_view str1, std::string_view str2)
309{
310 CaseInsensitiveStringView ci_str1{ str1.data(), str1.size() };
311 CaseInsensitiveStringView ci_str2{ str2.data(), str2.size() };
312 return ci_str1.compare(ci_str2);
313}
314
321bool StrEqualsIgnoreCase(std::string_view str1, std::string_view str2)
322{
323 if (str1.size() != str2.size()) return false;
324 return StrCompareIgnoreCase(str1, str2) == 0;
325}
326
334bool StrContainsIgnoreCase(std::string_view str, std::string_view value)
335{
336 CaseInsensitiveStringView ci_str{ str.data(), str.size() };
337 CaseInsensitiveStringView ci_value{ value.data(), value.size() };
338 return ci_str.find(ci_value) != ci_str.npos;
339}
340
347size_t Utf8StringLength(std::string_view str)
348{
349 Utf8View view(str);
350 return std::distance(view.begin(), view.end());
351}
352
353bool strtolower(std::string &str, std::string::size_type offs)
354{
355 bool changed = false;
356 for (auto ch = str.begin() + offs; ch != str.end(); ++ch) {
357 auto new_ch = static_cast<char>(tolower(static_cast<unsigned char>(*ch)));
358 changed |= new_ch != *ch;
359 *ch = new_ch;
360 }
361 return changed;
362}
363
371bool IsValidChar(char32_t key, CharSetFilter afilter)
372{
373 switch (afilter) {
374 case CS_ALPHANUMERAL: return IsPrintable(key);
375 case CS_NUMERAL: return (key >= '0' && key <= '9');
376 case CS_NUMERAL_SPACE: return (key >= '0' && key <= '9') || key == ' ';
377 case CS_NUMERAL_SIGNED: return (key >= '0' && key <= '9') || key == '-';
378 case CS_ALPHA: return IsPrintable(key) && !(key >= '0' && key <= '9');
379 case CS_HEXADECIMAL: return (key >= '0' && key <= '9') || (key >= 'a' && key <= 'f') || (key >= 'A' && key <= 'F');
380 default: NOT_REACHED();
381 }
382}
383
389static bool IsGarbageCharacter(char32_t c)
390{
391 if (c >= '0' && c <= '9') return false;
392 if (c >= 'A' && c <= 'Z') return false;
393 if (c >= 'a' && c <= 'z') return false;
394 if (c >= SCC_CONTROL_START && c <= SCC_CONTROL_END) return true;
395 if (c >= 0xC0 && c <= 0x10FFFF) return false;
396
397 return true;
398}
399
408static std::string_view SkipGarbage(std::string_view str)
409{
410 Utf8View view(str);
411 auto it = view.begin();
412 const auto end = view.end();
413 while (it != end && IsGarbageCharacter(*it)) ++it;
414 return str.substr(it.GetByteOffset());
415}
416
425int StrNaturalCompare(std::string_view s1, std::string_view s2, bool ignore_garbage_at_front)
426{
427 if (ignore_garbage_at_front) {
428 s1 = SkipGarbage(s1);
429 s2 = SkipGarbage(s2);
430 }
431
432#ifdef WITH_ICU_I18N
433 if (_current_collator) {
434 UErrorCode status = U_ZERO_ERROR;
435 int result = _current_collator->compareUTF8(icu::StringPiece(s1.data(), s1.size()), icu::StringPiece(s2.data(), s2.size()), status);
436 if (U_SUCCESS(status)) return result;
437 }
438#endif /* WITH_ICU_I18N */
439
440#if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
441 int res = OTTDStringCompare(s1, s2);
442 if (res != 0) return res - 2; // Convert to normal C return values.
443#endif
444
445#if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
446 int res = MacOSStringCompare(s1, s2);
447 if (res != 0) return res - 2; // Convert to normal C return values.
448#endif
449
450 /* Do a normal comparison if ICU is missing or if we cannot create a collator. */
451 return StrCompareIgnoreCase(s1, s2);
452}
453
454#ifdef WITH_ICU_I18N
455
456#include <unicode/stsearch.h>
457
466static int ICUStringContains(std::string_view str, std::string_view value, bool case_insensitive)
467{
468 if (_current_collator) {
469 std::unique_ptr<icu::RuleBasedCollator> coll(dynamic_cast<icu::RuleBasedCollator *>(_current_collator->clone()));
470 if (coll) {
471 UErrorCode status = U_ZERO_ERROR;
472 coll->setStrength(case_insensitive ? icu::Collator::SECONDARY : icu::Collator::TERTIARY);
473 coll->setAttribute(UCOL_NUMERIC_COLLATION, UCOL_OFF, status);
474
475 auto u_str = icu::UnicodeString::fromUTF8(icu::StringPiece(str.data(), str.size()));
476 auto u_value = icu::UnicodeString::fromUTF8(icu::StringPiece(value.data(), value.size()));
477 icu::StringSearch u_searcher(u_value, u_str, coll.get(), nullptr, status);
478 if (U_SUCCESS(status)) {
479 auto pos = u_searcher.first(status);
480 if (U_SUCCESS(status)) return pos != USEARCH_DONE ? 1 : 0;
481 }
482 }
483 }
484
485 return -1;
486}
487#endif /* WITH_ICU_I18N */
488
496[[nodiscard]] bool StrNaturalContains(std::string_view str, std::string_view value)
497{
498#ifdef WITH_ICU_I18N
499 int res_u = ICUStringContains(str, value, false);
500 if (res_u >= 0) return res_u > 0;
501#endif /* WITH_ICU_I18N */
502
503#if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
504 int res = Win32StringContains(str, value, false);
505 if (res >= 0) return res > 0;
506#endif
507
508#if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
509 int res = MacOSStringContains(str, value, false);
510 if (res >= 0) return res > 0;
511#endif
512
513 return str.find(value) != std::string_view::npos;
514}
515
523[[nodiscard]] bool StrNaturalContainsIgnoreCase(std::string_view str, std::string_view value)
524{
525#ifdef WITH_ICU_I18N
526 int res_u = ICUStringContains(str, value, true);
527 if (res_u >= 0) return res_u > 0;
528#endif /* WITH_ICU_I18N */
529
530#if defined(_WIN32) && !defined(STRGEN) && !defined(SETTINGSGEN)
531 int res = Win32StringContains(str, value, true);
532 if (res >= 0) return res > 0;
533#endif
534
535#if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
536 int res = MacOSStringContains(str, value, true);
537 if (res >= 0) return res > 0;
538#endif
539
540 CaseInsensitiveStringView ci_str{ str.data(), str.size() };
541 CaseInsensitiveStringView ci_value{ value.data(), value.size() };
542 return ci_str.find(ci_value) != CaseInsensitiveStringView::npos;
543}
544
551static int ConvertHexNibbleToByte(char c)
552{
553 if (c >= '0' && c <= '9') return c - '0';
554 if (c >= 'A' && c <= 'F') return c + 10 - 'A';
555 if (c >= 'a' && c <= 'f') return c + 10 - 'a';
556 return -1;
557}
558
570bool ConvertHexToBytes(std::string_view hex, std::span<uint8_t> bytes)
571{
572 if (bytes.size() != hex.size() / 2) {
573 return false;
574 }
575
576 /* Hex-string lengths are always divisible by 2. */
577 if (hex.size() % 2 != 0) {
578 return false;
579 }
580
581 for (size_t i = 0; i < hex.size() / 2; i++) {
582 auto hi = ConvertHexNibbleToByte(hex[i * 2]);
583 auto lo = ConvertHexNibbleToByte(hex[i * 2 + 1]);
584
585 if (hi < 0 || lo < 0) {
586 return false;
587 }
588
589 bytes[i] = (hi << 4) | lo;
590 }
591
592 return true;
593}
594
595#ifdef WITH_UNISCRIBE
596
597/* static */ std::unique_ptr<StringIterator> StringIterator::Create()
598{
599 return std::make_unique<UniscribeStringIterator>();
600}
601
602#elif defined(WITH_ICU_I18N)
603
604#include <unicode/utext.h>
605#include <unicode/brkiter.h>
606
609{
610 icu::BreakIterator *char_itr;
611 icu::BreakIterator *word_itr;
612
613 std::vector<UChar> utf16_str;
614 std::vector<size_t> utf16_to_utf8;
615
616public:
617 IcuStringIterator() : char_itr(nullptr), word_itr(nullptr)
618 {
619 UErrorCode status = U_ZERO_ERROR;
620 this->char_itr = icu::BreakIterator::createCharacterInstance(icu::Locale(_current_language != nullptr ? _current_language->isocode : "en"), status);
621 this->word_itr = icu::BreakIterator::createWordInstance(icu::Locale(_current_language != nullptr ? _current_language->isocode : "en"), status);
622
623 this->utf16_str.push_back('\0');
624 this->utf16_to_utf8.push_back(0);
625 }
626
627 ~IcuStringIterator() override
628 {
629 delete this->char_itr;
630 delete this->word_itr;
631 }
632
633 void SetString(std::string_view s) override
634 {
635 /* Unfortunately current ICU versions only provide rudimentary support
636 * for word break iterators (especially for CJK languages) in combination
637 * with UTF-8 input. As a work around we have to convert the input to
638 * UTF-16 and create a mapping back to UTF-8 character indices. */
639 this->utf16_str.clear();
640 this->utf16_to_utf8.clear();
641
642 Utf8View view(s);
643 for (auto it = view.begin(), end = view.end(); it != end; ++it) {
644 size_t idx = it.GetByteOffset();
645 char32_t c = *it;
646 if (c < 0x10000) {
647 this->utf16_str.push_back((UChar)c);
648 } else {
649 /* Make a surrogate pair. */
650 this->utf16_str.push_back((UChar)(0xD800 + ((c - 0x10000) >> 10)));
651 this->utf16_str.push_back((UChar)(0xDC00 + ((c - 0x10000) & 0x3FF)));
652 this->utf16_to_utf8.push_back(idx);
653 }
654 this->utf16_to_utf8.push_back(idx);
655 }
656 this->utf16_str.push_back('\0');
657 this->utf16_to_utf8.push_back(s.size());
658
659 UText text = UTEXT_INITIALIZER;
660 UErrorCode status = U_ZERO_ERROR;
661 utext_openUChars(&text, this->utf16_str.data(), this->utf16_str.size() - 1, &status);
662 this->char_itr->setText(&text, status);
663 this->word_itr->setText(&text, status);
664 this->char_itr->first();
665 this->word_itr->first();
666 }
667
668 size_t SetCurPosition(size_t pos) override
669 {
670 /* Convert incoming position to an UTF-16 string index. */
671 uint utf16_pos = 0;
672 for (uint i = 0; i < this->utf16_to_utf8.size(); i++) {
673 if (this->utf16_to_utf8[i] == pos) {
674 utf16_pos = i;
675 break;
676 }
677 }
678
679 /* isBoundary has the documented side-effect of setting the current
680 * position to the first valid boundary equal to or greater than
681 * the passed value. */
682 this->char_itr->isBoundary(utf16_pos);
683 return this->utf16_to_utf8[this->char_itr->current()];
684 }
685
686 size_t Next(IterType what) override
687 {
688 int32_t pos;
689 switch (what) {
690 case ITER_CHARACTER:
691 pos = this->char_itr->next();
692 break;
693
694 case ITER_WORD:
695 pos = this->word_itr->following(this->char_itr->current());
696 /* The ICU word iterator considers both the start and the end of a word a valid
697 * break point, but we only want word starts. Move to the next location in
698 * case the new position points to whitespace. */
699 while (pos != icu::BreakIterator::DONE &&
700 IsWhitespace(Utf16DecodeChar((const uint16_t *)&this->utf16_str[pos]))) {
701 int32_t new_pos = this->word_itr->next();
702 /* Don't set it to DONE if it was valid before. Otherwise we'll return END
703 * even though the iterator wasn't at the end of the string before. */
704 if (new_pos == icu::BreakIterator::DONE) break;
705 pos = new_pos;
706 }
707
708 this->char_itr->isBoundary(pos);
709 break;
710
711 default:
712 NOT_REACHED();
713 }
714
715 return pos == icu::BreakIterator::DONE ? END : this->utf16_to_utf8[pos];
716 }
717
718 size_t Prev(IterType what) override
719 {
720 int32_t pos;
721 switch (what) {
722 case ITER_CHARACTER:
723 pos = this->char_itr->previous();
724 break;
725
726 case ITER_WORD:
727 pos = this->word_itr->preceding(this->char_itr->current());
728 /* The ICU word iterator considers both the start and the end of a word a valid
729 * break point, but we only want word starts. Move to the previous location in
730 * case the new position points to whitespace. */
731 while (pos != icu::BreakIterator::DONE &&
732 IsWhitespace(Utf16DecodeChar((const uint16_t *)&this->utf16_str[pos]))) {
733 int32_t new_pos = this->word_itr->previous();
734 /* Don't set it to DONE if it was valid before. Otherwise we'll return END
735 * even though the iterator wasn't at the start of the string before. */
736 if (new_pos == icu::BreakIterator::DONE) break;
737 pos = new_pos;
738 }
739
740 this->char_itr->isBoundary(pos);
741 break;
742
743 default:
744 NOT_REACHED();
745 }
746
747 return pos == icu::BreakIterator::DONE ? END : this->utf16_to_utf8[pos];
748 }
749};
750
751/* static */ std::unique_ptr<StringIterator> StringIterator::Create()
752{
753 return std::make_unique<IcuStringIterator>();
754}
755
756#else
757
759class DefaultStringIterator : public StringIterator
760{
761 Utf8View string;
762 Utf8View::iterator cur_pos; //< Current iteration position.
763
764public:
765 void SetString(std::string_view s) override
766 {
767 this->string = s;
768 this->cur_pos = this->string.begin();
769 }
770
771 size_t SetCurPosition(size_t pos) override
772 {
773 this->cur_pos = this->string.GetIterAtByte(pos);
774 return this->cur_pos.GetByteOffset();
775 }
776
777 size_t Next(IterType what) override
778 {
779 const auto end = this->string.end();
780 /* Already at the end? */
781 if (this->cur_pos >= end) return END;
782
783 switch (what) {
784 case ITER_CHARACTER:
785 ++this->cur_pos;
786 return this->cur_pos.GetByteOffset();
787
788 case ITER_WORD:
789 /* Consume current word. */
790 while (this->cur_pos != end && !IsWhitespace(*this->cur_pos)) {
791 ++this->cur_pos;
792 }
793 /* Consume whitespace to the next word. */
794 while (this->cur_pos != end && IsWhitespace(*this->cur_pos)) {
795 ++this->cur_pos;
796 }
797 return this->cur_pos.GetByteOffset();
798
799 default:
800 NOT_REACHED();
801 }
802
803 return END;
804 }
805
806 size_t Prev(IterType what) override
807 {
808 const auto begin = this->string.begin();
809 /* Already at the beginning? */
810 if (this->cur_pos == begin) return END;
811
812 switch (what) {
813 case ITER_CHARACTER:
814 --this->cur_pos;
815 return this->cur_pos.GetByteOffset();
816
817 case ITER_WORD:
818 /* Consume preceding whitespace. */
819 do {
820 --this->cur_pos;
821 } while (this->cur_pos != begin && IsWhitespace(*this->cur_pos));
822 /* Consume preceding word. */
823 while (this->cur_pos != begin && !IsWhitespace(*this->cur_pos)) {
824 --this->cur_pos;
825 }
826 /* Move caret back to the beginning of the word. */
827 if (IsWhitespace(*this->cur_pos)) ++this->cur_pos;
828 return this->cur_pos.GetByteOffset();
829
830 default:
831 NOT_REACHED();
832 }
833
834 return END;
835 }
836};
837
838#if defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN)
839/* static */ std::unique_ptr<StringIterator> StringIterator::Create()
840{
841 std::unique_ptr<StringIterator> i = OSXStringIterator::Create();
842 if (i != nullptr) return i;
843
844 return std::make_unique<DefaultStringIterator>();
845}
846#else
847/* static */ std::unique_ptr<StringIterator> StringIterator::Create()
848{
849 return std::make_unique<DefaultStringIterator>();
850}
851#endif /* defined(WITH_COCOA) && !defined(STRGEN) && !defined(SETTINGSGEN) */
852
853#endif
854
860std::optional<std::string_view> GetEnv(const char *variable)
861{
862 auto val = std::getenv(variable);
863 if (val == nullptr || *val == '\0') return std::nullopt;
864 return val;
865}
void PutChar(char c)
Append 8-bit char.
Enum-as-bit-set wrapper.
String iterator using ICU as a backend.
Definition string.cpp:609
size_t Prev(IterType what) override
Move the cursor back by one iteration unit.
Definition string.cpp:718
size_t Next(IterType what) override
Advance the cursor by one iteration unit.
Definition string.cpp:686
std::vector< size_t > utf16_to_utf8
Mapping from UTF-16 code point position to index in the UTF-8 source string.
Definition string.cpp:614
void SetString(std::string_view s) override
Set a new iteration string.
Definition string.cpp:633
size_t SetCurPosition(size_t pos) override
Change the current string cursor.
Definition string.cpp:668
std::vector< UChar > utf16_str
UTF-16 copy of the string.
Definition string.cpp:613
icu::BreakIterator * char_itr
ICU iterator for characters.
Definition string.cpp:610
icu::BreakIterator * word_itr
ICU iterator for words.
Definition string.cpp:611
bool AnyBytesUnused() const noexcept
Check whether any unused bytes are left between the Builder and Consumer position.
size_type GetBytesWritten() const noexcept
Get number of already written bytes.
Compose data into a fixed size buffer, which is consumed at the same time.
InPlaceBuilder builder
Builder into shared buffer.
StringConsumer consumer
Consumer from shared buffer.
Compose data into a growing std::string.
Parse data from a string / buffer.
bool AnyBytesLeft() const noexcept
Check whether any bytes left to read.
bool PeekCharIf(char c) const
Check whether the next 8-bit char matches 'c'.
static const std::string_view WHITESPACE_NO_NEWLINE
ASCII whitespace characters, excluding new-line.
std::optional< char32_t > TryReadUtf8()
Try to read a UTF-8 character, and then advance reader.
void Skip(size_type len)
Discard some bytes.
Class for iterating over different kind of parts of a string.
Definition string_base.h:14
static const size_t END
Sentinel to indicate end-of-iteration.
Definition string_base.h:23
virtual size_t Prev(IterType what=ITER_CHARACTER)=0
Move the cursor back by one iteration unit.
virtual size_t SetCurPosition(size_t pos)=0
Change the current string cursor.
virtual size_t Next(IterType what=ITER_CHARACTER)=0
Advance the cursor by one iteration unit.
static std::unique_ptr< StringIterator > Create()
Create a new iterator instance.
Definition string.cpp:751
IterType
Type of the iterator.
Definition string_base.h:17
@ ITER_WORD
Iterate over words.
Definition string_base.h:19
@ ITER_CHARACTER
Iterate over characters (or more exactly grapheme clusters).
Definition string_base.h:18
virtual void SetString(std::string_view s)=0
Set a new iteration string.
Bidirectional input iterator over codepoints.
Definition utf8.hpp:43
Constant span of UTF-8 encoded data.
Definition utf8.hpp:30
Control codes that are embedded in the translation strings.
@ SCC_ENCODED
Encoded string marker and sub-string parameter.
@ SCC_ENCODED_NUMERIC
Encoded numeric parameter.
@ SCC_ENCODED_STRING
Encoded string parameter.
@ SCC_ENCODED_INTERNAL
Encoded text from OpenTTD.
Functions related to debugging.
#define Debug(category, level, format_string,...)
Output a line of debugging information.
Definition debug.h:37
Error reporting related functions.
fluid_settings_t * settings
FluidSynth settings handle.
Functions related to the gfx engine.
Information about languages and their files.
const LanguageMetadata * _current_language
The currently loaded language.
Definition strings.cpp:55
std::unique_ptr< icu::Collator > _current_collator
Collator for the language currently in use.
Definition strings.cpp:60
Integer math functions.
A number of safeguards to prevent using unsafe methods.
Definition of base types and functions in a cross-platform compatible way.
bool ConvertHexToBytes(std::string_view hex, std::span< uint8_t > bytes)
Convert a hex-string to a byte-array, while validating it was actually hex.
Definition string.cpp:570
bool StrNaturalContainsIgnoreCase(std::string_view str, std::string_view value)
Checks if a string is contained in another string with a locale-aware comparison that is case insensi...
Definition string.cpp:523
size_t Utf8StringLength(std::string_view str)
Get the length of an UTF-8 encoded string in number of characters and thus not the number of bytes th...
Definition string.cpp:347
static int ICUStringContains(std::string_view str, std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
Definition string.cpp:466
bool StrNaturalContains(std::string_view str, std::string_view value)
Checks if a string is contained in another string with a locale-aware comparison that is case sensiti...
Definition string.cpp:496
bool IsValidChar(char32_t key, CharSetFilter afilter)
Only allow certain keys.
Definition string.cpp:371
std::optional< std::string_view > GetEnv(const char *variable)
Get the environment variable using std::getenv and when it is an empty string (or nullptr),...
Definition string.cpp:860
void StrMakeValidInPlace(char *str, StringValidationSettings settings)
Scans the string for invalid characters and replaces them with a question mark '?' (if not ignored).
Definition string.cpp:155
void strecpy(std::span< char > dst, std::string_view src)
Copies characters from one buffer to another.
Definition string.cpp:54
std::string FormatArrayAsHex(std::span< const uint8_t > data)
Format a byte array into a continuous hex string.
Definition string.cpp:75
bool StrEqualsIgnoreCase(std::string_view str1, std::string_view str2)
Compares two string( view)s for equality, while ignoring the case of the characters.
Definition string.cpp:321
bool StrEndsWithIgnoreCase(std::string_view str, std::string_view suffix)
Check whether the given string ends with the given suffix, ignoring case.
Definition string.cpp:295
bool StrValid(std::span< const char > str)
Checks whether the given string is valid, i.e.
Definition string.cpp:203
static int ConvertHexNibbleToByte(char c)
Convert a single hex-nibble to a byte.
Definition string.cpp:551
static std::string_view SkipGarbage(std::string_view str)
Skip some of the 'garbage' in the string that we don't want to use to sort on.
Definition string.cpp:408
static bool IsSccEncodedCode(char32_t c)
Test if a character is (only) part of an encoded string.
Definition string.cpp:92
int StrNaturalCompare(std::string_view s1, std::string_view s2, bool ignore_garbage_at_front)
Compares two strings using case insensitive natural sort.
Definition string.cpp:425
std::basic_string_view< char, CaseInsensitiveCharTraits > CaseInsensitiveStringView
Case insensitive string view.
Definition string.cpp:287
void StrTrimInPlace(std::string &str)
Trim the spaces from given string in place, i.e.
Definition string.cpp:226
bool StrStartsWithIgnoreCase(std::string_view str, std::string_view prefix)
Check whether the given string starts with the given prefix, ignoring case.
Definition string.cpp:255
static void StrMakeValid(Builder &builder, StringConsumer &consumer, StringValidationSettings settings)
Copies the valid (UTF-8) characters from consumer to the builder.
Definition string.cpp:117
int StrCompareIgnoreCase(std::string_view str1, std::string_view str2)
Compares two string( view)s, while ignoring the case of the characters.
Definition string.cpp:308
static bool IsGarbageCharacter(char32_t c)
Test if a unicode character is considered garbage to be skipped.
Definition string.cpp:389
bool StrContainsIgnoreCase(std::string_view str, std::string_view value)
Checks if a string is contained in another string, while ignoring the case of the characters.
Definition string.cpp:334
Functions related to low-level strings.
char32_t Utf16DecodeChar(const uint16_t *c)
Decode an UTF-16 character.
Definition string_func.h:97
bool IsWhitespace(char32_t c)
Check whether UNICODE character is whitespace or not, i.e.
Inplace-replacement of textual and binary data.
int MacOSStringCompare(std::string_view s1, std::string_view s2)
Compares two strings using case insensitive natural sort.
int MacOSStringContains(std::string_view str, std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
Functions related to localized text support on OSX.
@ ReplaceWithQuestionMark
Replace the unknown/bad bits with question marks.
@ AllowControlCode
Allow the special control codes.
@ AllowNewline
Allow newlines; replaces '\r ' with ' ' during processing.
@ ReplaceTabCrNlWithSpace
Replace tabs ('\t'), carriage returns ('\r') and newlines (' ') with spaces.
CharSetFilter
Valid filter types for IsValidChar.
Definition string_type.h:24
@ CS_NUMERAL_SPACE
Only numbers and spaces.
Definition string_type.h:27
@ CS_HEXADECIMAL
Only hexadecimal characters.
Definition string_type.h:30
@ CS_NUMERAL
Only numeric ones.
Definition string_type.h:26
@ CS_NUMERAL_SIGNED
Only numbers and '-' for negative values.
Definition string_type.h:28
@ CS_ALPHA
Only alphabetic values.
Definition string_type.h:29
@ CS_ALPHANUMERAL
Both numeric and alphabetic and spaces and stuff.
Definition string_type.h:25
Functions related to laying out text on Win32.
Case insensitive implementation of the standard character type traits.
Definition string.cpp:262
char isocode[16]
the ISO code for the language (not country code)
Definition language.h:31
Handling of UTF-8 encoded data.
int Win32StringContains(std::string_view str, std::string_view value, bool case_insensitive)
Search if a string is contained in another string using the current locale.
Definition win32.cpp:477
declarations of functions for MS windows systems