OpenTTD Source 20261112-master-gbcbec5e63f
gfx_layout_icu.cpp
Go to the documentation of this file.
1/*
2 * This file is part of OpenTTD.
3 * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
4 * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
5 * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <https://www.gnu.org/licenses/old-licenses/gpl-2.0>.
6 */
7
9
10#include "stdafx.h"
11#include "gfx_layout_icu.h"
12
13#include "debug.h"
14#include "misc/autorelease.hpp"
15#include "strings_func.h"
16#include "language.h"
17#include "table/control_codes.h"
18#include "zoom_func.h"
19
20#include "3rdparty/icu/scriptrun.h"
21
22#include <unicode/ubidi.h>
23#include <unicode/brkiter.h>
24
25#include <hb.h>
26#include <hb-ft.h>
27
28#include "safeguards.h"
29
31constexpr float FONT_SCALE = 64.0;
32
38class ICURun {
39public:
40 int start;
41 int length;
42 UBiDiLevel level;
43 UScriptCode script;
45
46 std::vector<GlyphID> glyphs;
47 std::vector<int> advance;
48 std::vector<int> glyph_to_char;
49 std::vector<ParagraphLayouter::Position> positions;
50 int total_advance = 0;
51
60 ICURun(int start, int length, UBiDiLevel level, UScriptCode script, const Font &font) : start(start), length(length), level(level), script(script), font(font) {}
61
62 void Shape(UChar *buff, size_t length);
63};
64
68class ICUParagraphLayout : public ParagraphLayouter {
69public:
72 private:
73 std::vector<GlyphID> glyphs;
74 std::vector<Position> positions;
75 std::vector<int> glyph_to_char;
76
77 int total_advance;
78 Font font;
79
80 public:
81 ICUVisualRun(const ICURun &run, int x);
82
83 std::span<const GlyphID> GetGlyphs() const override { return this->glyphs; }
84 std::span<const Position> GetPositions() const override { return this->positions; }
85 std::span<const int> GetGlyphToCharMap() const override { return this->glyph_to_char; }
86
87 const Font &GetFont() const override { return this->font; }
88 int GetLeading() const override { return this->font.GetFontCache().GetHeight(); }
89 size_t GetGlyphCount() const override { return this->glyphs.size(); }
90 int GetAdvance() const { return this->total_advance; }
91 };
92
94 class ICULine : public std::vector<ICUVisualRun>, public ParagraphLayouter::Line {
95 public:
96 int GetLeading() const override;
97 int GetWidth() const override;
98 size_t CountRuns() const override { return this->size(); }
99 const VisualRun &GetVisualRun(size_t run) const override { return this->at(run); }
100
101 int GetInternalCharLength(char32_t c) const override
102 {
103 /* ICU uses UTF-16 internally which means we need to account for surrogate pairs. */
104 return c >= 0x010000U ? 2 : 1;
105 }
106 };
107
108private:
109 std::vector<ICURun> runs;
110 UChar *buff;
111 size_t buff_length;
112 std::vector<ICURun>::iterator current_run;
113 int partial_offset;
114
115public:
116 ICUParagraphLayout(std::vector<ICURun> &&runs, UChar *buff, size_t buff_length) : runs(std::move(runs)), buff(buff), buff_length(buff_length)
117 {
118 this->Reflow();
119 }
120
121 ~ICUParagraphLayout() override = default;
122
123 void Reflow() override
124 {
125 this->current_run = this->runs.begin();
126 this->partial_offset = 0;
127 }
128
129 std::unique_ptr<const Line> NextLine(int max_width) override;
130};
131
141 glyphs(run.glyphs), glyph_to_char(run.glyph_to_char), total_advance(run.total_advance), font(run.font)
142{
143 /* If there are no positions, the ICURun was not Shaped; that should never happen. */
144 assert(!run.positions.empty());
145 this->positions.reserve(run.positions.size());
146
147 /* Copy positions, moving x coordinate by x offset. */
148 for (const auto &pos : run.positions) {
149 this->positions.emplace_back(pos.left + x, pos.right + x, pos.top);
150 }
151}
152
159void ICURun::Shape(UChar *buff, size_t buff_length)
160{
161 FontCache &fc = this->font.GetFontCache();
162 assert(fc.GetOSHandle() != nullptr);
163
164 auto hbfont = hb_ft_font_create_referenced(*(static_cast<const FT_Face *>(fc.GetOSHandle())));
165 /* Match the flags with how we render the glyphs. */
166 hb_ft_font_set_load_flags(hbfont, GetFontAAState() ? FT_LOAD_TARGET_NORMAL : FT_LOAD_TARGET_MONO);
167
168 /* ICU buffer is in UTF-16. */
169 auto hbbuf = hb_buffer_create();
170 hb_buffer_add_utf16(hbbuf, reinterpret_cast<uint16_t *>(buff), buff_length, this->start, this->length);
171
172 /* Set all the properties of this segment. */
173 hb_buffer_set_direction(hbbuf, (this->level & 1) == 1 ? HB_DIRECTION_RTL : HB_DIRECTION_LTR);
174 hb_buffer_set_script(hbbuf, hb_script_from_string(uscript_getShortName(this->script), -1));
175 hb_buffer_set_language(hbbuf, hb_language_from_string(_current_language->isocode, -1));
176 hb_buffer_set_cluster_level(hbbuf, HB_BUFFER_CLUSTER_LEVEL_MONOTONE_GRAPHEMES);
177
178 /* Shape the segment. */
179 hb_shape(hbfont, hbbuf, nullptr, 0);
180
181 unsigned int glyph_count;
182 auto glyph_info = hb_buffer_get_glyph_infos(hbbuf, &glyph_count);
183 auto glyph_pos = hb_buffer_get_glyph_positions(hbbuf, &glyph_count);
184
185 /* Make sure any former run is lost. */
186 this->glyphs.clear();
187 this->glyph_to_char.clear();
188 this->positions.clear();
189 this->advance.clear();
190
191 /* Reserve space, as we already know the size. */
192 this->glyphs.reserve(glyph_count);
193 this->glyph_to_char.reserve(glyph_count);
194 this->positions.reserve(glyph_count);
195 this->advance.reserve(glyph_count);
196
197 /* Prepare the glyphs/position. ICUVisualRun will give the position an offset if needed. */
198 hb_position_t advance = 0;
199 for (unsigned int i = 0; i < glyph_count; i++) {
200 int x_advance;
201
202 if (buff[glyph_info[i].cluster] >= SCC_SPRITE_START && buff[glyph_info[i].cluster] <= SCC_SPRITE_END && glyph_info[i].codepoint == 0) {
203 auto glyph = fc.MapCharToGlyph(buff[glyph_info[i].cluster]);
204 x_advance = fc.GetGlyphWidth(glyph);
205 this->glyphs.push_back(glyph);
206 this->positions.emplace_back(advance, advance + x_advance - 1, (fc.GetHeight() - ScaleSpriteTrad(FontCache::GetDefaultFontHeight(fc.GetSize()))) / 2); // Align sprite font to centre
207 } else {
208 x_advance = glyph_pos[i].x_advance / FONT_SCALE;
209 this->glyphs.push_back(glyph_info[i].codepoint);
210 this->positions.emplace_back(glyph_pos[i].x_offset / FONT_SCALE + advance, glyph_pos[i].x_offset / FONT_SCALE + advance + x_advance - 1, glyph_pos[i].y_offset / FONT_SCALE);
211 }
212
213 this->glyph_to_char.push_back(glyph_info[i].cluster);
214 this->advance.push_back(x_advance);
215 advance += x_advance;
216 }
217
218 /* Track the total advancement we made. */
219 this->total_advance = advance;
220
221 hb_buffer_destroy(hbbuf);
222 hb_font_destroy(hbfont);
223}
224
230{
231 int leading = 0;
232 for (const auto &run : *this) {
233 leading = std::max(leading, run.GetLeading());
234 }
235
236 return leading;
237}
238
244{
245 int length = 0;
246 for (const auto &run : *this) {
247 length += run.GetAdvance();
248 }
249
250 return length;
251}
252
262std::vector<ICURun> ItemizeBidi(UChar *buff, size_t length)
263{
264 auto ubidi = AutoRelease<UBiDi, ubidi_close>(ubidi_open());
265
266 auto parLevel = _current_text_dir == TD_RTL ? UBIDI_RTL : UBIDI_LTR;
267
268 UErrorCode err = U_ZERO_ERROR;
269 ubidi_setPara(ubidi.get(), buff, length, parLevel, nullptr, &err);
270 if (U_FAILURE(err)) {
271 Debug(Facility::Fontcache, Severity::Critical, "Failed to set paragraph: {}", u_errorName(err));
272 return {};
273 }
274
275 int32_t count = ubidi_countRuns(ubidi.get(), &err);
276 if (U_FAILURE(err)) {
277 Debug(Facility::Fontcache, Severity::Critical, "Failed to count runs: {}", u_errorName(err));
278 return {};
279 }
280
281 std::vector<ICURun> runs;
282 runs.reserve(count);
283
284 /* Find the breakpoints for the logical runs. So we get runs that say "from START to END". */
285 int32_t logical_pos = 0;
286 while (static_cast<size_t>(logical_pos) < length) {
287 auto start_pos = logical_pos;
288
289 /* Fetch the embedding level, so we can order bidi correctly later on. */
290 UBiDiLevel level;
291 ubidi_getLogicalRun(ubidi.get(), start_pos, &logical_pos, &level);
292
293 runs.emplace_back(start_pos, logical_pos - start_pos, level, USCRIPT_UNKNOWN, Font{});
294 }
295
296 assert(static_cast<size_t>(count) == runs.size());
297
298 return runs;
299}
300
311std::vector<ICURun> ItemizeScript(UChar *buff, size_t length, std::vector<ICURun> &runs_current)
312{
313 std::vector<ICURun> runs;
314 icu::ScriptRun script_itemizer(buff, length);
315
316 int cur_pos = 0;
317 auto cur_run = runs_current.begin();
318 while (true) {
319 while (cur_pos < script_itemizer.getScriptEnd() && cur_run != runs_current.end()) {
320 int stop_pos = std::min(script_itemizer.getScriptEnd(), cur_run->start + cur_run->length);
321 assert(stop_pos - cur_pos > 0);
322
323 runs.emplace_back(cur_pos, stop_pos - cur_pos, cur_run->level, script_itemizer.getScriptCode(), Font{});
324
325 if (stop_pos == cur_run->start + cur_run->length) cur_run++;
326 cur_pos = stop_pos;
327 }
328
329 if (!script_itemizer.next()) break;
330 }
331
332 return runs;
333}
334
344std::vector<ICURun> ItemizeStyle(std::vector<ICURun> &runs_current, FontMap &font_mapping)
345{
346 std::vector<ICURun> runs;
347
348 int cur_pos = 0;
349 auto cur_run = runs_current.begin();
350 for (auto const &[position, font] : font_mapping) {
351 while (cur_pos < position && cur_run != runs_current.end()) {
352 int stop_pos = std::min(position, cur_run->start + cur_run->length);
353 assert(stop_pos - cur_pos > 0);
354
355 runs.emplace_back(cur_pos, stop_pos - cur_pos, cur_run->level, cur_run->script, font);
356
357 if (stop_pos == cur_run->start + cur_run->length) cur_run++;
358 cur_pos = stop_pos;
359 }
360 }
361
362 return runs;
363}
364
365/* static */ std::unique_ptr<ParagraphLayouter> ICUParagraphLayoutFactory::GetParagraphLayout(UChar *buff, UChar *buff_end, FontMap &font_mapping)
366{
367 size_t length = buff_end - buff;
368 /* Can't layout an empty string. */
369 if (length == 0) return nullptr;
370
371 /* Can't layout our in-built sprite fonts. */
372 for (auto const &[position, font] : font_mapping) {
373 if (font.GetFontCache().IsBuiltInFont()) return nullptr;
374 }
375
376 auto runs = ItemizeBidi(buff, length);
377 runs = ItemizeScript(buff, length, runs);
378 runs = ItemizeStyle(runs, font_mapping);
379
380 if (runs.empty()) return nullptr;
381
382 for (auto &run : runs) {
383 run.Shape(buff, length);
384 }
385
386 return std::make_unique<ICUParagraphLayout>(std::move(runs), buff, length);
387}
388
389/* static */ std::unique_ptr<icu::BreakIterator> ICUParagraphLayoutFactory::break_iterator;
390
395{
396 auto locale = icu::Locale(_current_language->isocode);
397 UErrorCode status = U_ZERO_ERROR;
398 ICUParagraphLayoutFactory::break_iterator.reset(icu::BreakIterator::createLineInstance(locale, status));
399 assert(U_SUCCESS(status));
400}
401
406/* static */ std::unique_ptr<icu::BreakIterator> ICUParagraphLayoutFactory::GetBreakIterator()
407{
408 assert(ICUParagraphLayoutFactory::break_iterator != nullptr);
409
410 return std::unique_ptr<icu::BreakIterator>(ICUParagraphLayoutFactory::break_iterator->clone());
411}
412
413std::unique_ptr<const ICUParagraphLayout::Line> ICUParagraphLayout::NextLine(int max_width)
414{
415 std::vector<ICURun>::iterator start_run = this->current_run;
416 std::vector<ICURun>::iterator last_run = this->current_run;
417
418 if (start_run == this->runs.end()) return nullptr;
419
420 int cur_width = 0;
421
422 /* Add remaining width of the first run if it is a broken run. */
423 if (this->partial_offset > 0) {
424 if ((start_run->level & 1) == 0) {
425 for (size_t i = this->partial_offset; i < start_run->advance.size(); i++) {
426 cur_width += start_run->advance[i];
427 }
428 } else {
429 for (int i = 0; i < this->partial_offset; i++) {
430 cur_width += start_run->advance[i];
431 }
432 }
433 last_run++;
434 }
435
436 /* Gather runs until the line is full. */
437 while (last_run != this->runs.end() && cur_width < max_width) {
438 cur_width += last_run->total_advance;
439 last_run++;
440 }
441
442 /* If the text does not fit into the available width, find a suitable breaking point. */
443 int new_partial_length = 0;
444 if (cur_width > max_width) {
445 /* Create a break-iterator to find a good place to break lines. */
446 auto break_iterator = ICUParagraphLayoutFactory::GetBreakIterator();
447 icu::UnicodeString text(this->buff, this->buff_length);
448 break_iterator->setText(text);
449
450 auto overflow_run = last_run - 1;
451
452 /* Find the last glyph that fits. */
453 size_t index;
454 if ((overflow_run->level & 1) == 0) {
455 /* LTR */
456 for (index = overflow_run->glyphs.size(); index > 0; /* nothing */) {
457 --index;
458 cur_width -= overflow_run->advance[index];
459 if (cur_width <= max_width) break;
460 }
461 } else {
462 /* RTL */
463 for (index = 0; index < overflow_run->glyphs.size(); index++) {
464 cur_width -= overflow_run->advance[index];
465 if (cur_width <= max_width) break;
466 }
467 }
468
469 /* Find the character that matches; this is the start of the cluster. */
470 auto char_pos = overflow_run->glyph_to_char[index];
471
472 /* See if there is a good breakpoint inside this run. */
473 int32_t break_pos = break_iterator->preceding(char_pos + 1);
474 auto overflow_run_start = overflow_run->start;
475 if (overflow_run == start_run) overflow_run_start += this->partial_offset;
476 if (break_pos != icu::BreakIterator::DONE && break_pos > overflow_run_start) {
477 /* There is a line-break inside this run that is suitable. */
478 new_partial_length = break_pos - overflow_run_start;
479 } else if (overflow_run != start_run) {
480 /* There is no suitable line-break in this run, but it is also not
481 * the only run on this line. So we remove the run. */
482 last_run--;
483 } else {
484 /* There is no suitable line-break and this is the only run on the
485 * line. So we break at the cluster. This is not pretty, but the
486 * best we can do. */
487 new_partial_length = char_pos - overflow_run_start;
488 }
489 }
490
491 /* Reorder the runs on this line for display. */
492 std::vector<UBiDiLevel> bidi_level;
493 for (auto run = start_run; run != last_run; run++) {
494 bidi_level.push_back(run->level);
495 }
496 std::vector<int32_t> vis_to_log(bidi_level.size());
497 ubidi_reorderVisual(bidi_level.data(), bidi_level.size(), vis_to_log.data());
498
499 /* Create line. */
500 std::unique_ptr<ICULine> line = std::make_unique<ICULine>();
501
502 int cur_pos = 0;
503 for (auto &i : vis_to_log) {
504 auto i_run = start_run + i;
505 /* Copy the ICURun here, so we can modify it in case of a partial. */
506 ICURun run = *i_run;
507
508 if (i_run == last_run - 1 && new_partial_length > 0) {
509 if (i_run == start_run && this->partial_offset > 0) {
510 assert(run.length > this->partial_offset);
511 run.start += this->partial_offset;
512 run.length -= this->partial_offset;
513 }
514
515 assert(run.length > new_partial_length);
516 run.length = new_partial_length;
517
518 run.Shape(this->buff, this->buff_length);
519 } else if (i_run == start_run && this->partial_offset > 0) {
520 assert(run.length > this->partial_offset);
521
522 run.start += this->partial_offset;
523 run.length -= this->partial_offset;
524
525 run.Shape(this->buff, this->buff_length);
526 }
527
528 auto total_advance = run.total_advance;
529 line->emplace_back(std::move(run), cur_pos);
530 cur_pos += total_advance;
531 }
532
533 if (new_partial_length > 0) {
534 this->current_run = last_run - 1;
535 if (this->current_run != start_run) this->partial_offset = 0;
536 this->partial_offset += new_partial_length;
537 } else {
538 this->current_run = last_run;
539 this->partial_offset = 0;
540 }
541
542 return line;
543}
544
545/* static */ size_t ICUParagraphLayoutFactory::AppendToBuffer(UChar *buff, const UChar *buffer_last, char32_t c)
546{
547 assert(buff < buffer_last);
548 /* Transform from UTF-32 to internal ICU format of UTF-16. */
549 int32_t length = 0;
550 UErrorCode err = U_ZERO_ERROR;
551 u_strFromUTF32(buff, buffer_last - buff, &length, (UChar32*)&c, 1, &err);
552 return length;
553}
Helper for std::unique_ptr to use an arbitrary function as the deleter.
std::unique_ptr< T, DeleterFromFunc< Tfunc > > AutoRelease
Specialisation of std::unique_ptr for objects which must be deleted by calling a function.
Font cache for basic fonts.
Definition fontcache.h:23
int GetHeight() const
Get the height of the font.
Definition fontcache.h:65
virtual const void * GetOSHandle()
Get the native OS font handle, if there is one.
Definition fontcache.h:120
virtual GlyphID MapCharToGlyph(char32_t key, bool fallback=true)=0
Map a character into a glyph.
virtual uint GetGlyphWidth(GlyphID key)=0
Get the width of the glyph with the given key.
FontSize GetSize() const
Get the FontSize of the font.
Definition fontcache.h:59
Container with information about a font.
Definition gfx_layout.h:21
FontCache & GetFontCache() const
Get the font cache of this font.
Definition gfx_layout.h:36
static std::unique_ptr< icu::BreakIterator > GetBreakIterator()
Get a thread-safe line break iterator.
static void InitializeLayouter()
Initialize data needed for the ICU layouter.
A single line worth of VisualRuns.
int GetInternalCharLength(char32_t c) const override
Get the number of elements the given character occupies in the underlying text buffer of the Layouter...
const VisualRun & GetVisualRun(size_t run) const override
Get a reference to the given run.
int GetLeading() const override
Get the height of the line.
int GetWidth() const override
Get the width of this line.
size_t CountRuns() const override
Get the number of runs in this line.
std::span< const Position > GetPositions() const override
Get the positions for each of the glyphs.
ICUVisualRun(const ICURun &run, int x)
Constructor for a new ICUVisualRun.
int GetLeading() const override
Get the font leading, or distance between the baselines of consecutive lines.
std::span< const GlyphID > GetGlyphs() const override
Get the glyphs to draw.
size_t GetGlyphCount() const override
Get the number of glyphs.
std::span< const int > GetGlyphToCharMap() const override
The offset for each of the glyphs to the character run that was passed to the Layouter.
const Font & GetFont() const override
Get the font.
Wrapper for doing layouts with ICU.
void Reflow() override
Reset the position to the start of the paragraph.
std::unique_ptr< const Line > NextLine(int max_width) override
Construct a new line with a maximum width.
Helper class to store the information of all the runs of a paragraph in.
UScriptCode script
Script of the run.
Font font
Font of the run.
std::vector< int > glyph_to_char
The mapping from glyphs to characters. Valid after Shape() is called.
ICURun(int start, int length, UBiDiLevel level, UScriptCode script, const Font &font)
Construct a new ICURun object.
void Shape(UChar *buff, size_t length)
Shape a single run.
std::vector< GlyphID > glyphs
The glyphs of the run. Valid after Shape() is called.
int total_advance
The total advance of the run. Valid after Shape() is called.
std::vector< int > advance
The advance (width) of the glyphs. Valid after Shape() is called.
int length
Length of the run in the buffer.
int start
Start of the run in the buffer.
std::vector< ParagraphLayouter::Position > positions
The positions of the glyphs. Valid after Shape() is called.
UBiDiLevel level
Embedding level of the run.
A single line worth of VisualRuns.
Definition gfx_layout.h:209
Visual run contains data about the bit of text with the same font.
Definition gfx_layout.h:166
Interface to glue fallback and normal layouter into one.
Definition gfx_layout.h:144
Control codes that are embedded in the translation strings.
Functions related to debugging.
#define Debug(facility, severity, format_string,...)
Output a line of debugging information.
Definition debug.h:36
@ Fontcache
Fontcache message facility.
Definition debug_type.h:37
@ Critical
Critical, user should know about this.
Definition debug_type.h:15
std::vector< std::pair< int, Font > > FontMap
Mapping from index to font.
Definition gfx_layout.h:139
std::vector< ICURun > ItemizeStyle(std::vector< ICURun > &runs_current, FontMap &font_mapping)
Itemize the string into runs per style, based on the previous created runs.
constexpr float FONT_SCALE
HarfBuzz FreeType integration sets the font scaling, which is always in 1/64th of a pixel.
std::vector< ICURun > ItemizeBidi(UChar *buff, size_t length)
Itemize the string into runs per embedding level.
std::vector< ICURun > ItemizeScript(UChar *buff, size_t length, std::vector< ICURun > &runs_current)
Itemize the string into runs per script, based on the previous created runs.
Functions related to laying out the texts with ICU.
Information about languages and their files.
const LanguageMetadata * _current_language
The currently loaded language.
Definition strings.cpp:53
A number of safeguards to prevent using unsafe methods.
Definition of base types and functions in a cross-platform compatible way.
TextDirection _current_text_dir
Text direction of the currently selected language.
Definition strings.cpp:55
Functions related to OTTD's strings.
@ TD_RTL
Text is written right-to-left by default.
Functions related to zooming.
int ScaleSpriteTrad(int value)
Scale traditional pixel dimensions to GUI zoom level, for drawing sprites.
Definition zoom_func.h:107