Skip to content

Commit aa99ac2

Browse files
gershnikgershnik
committed
Making python also use new ct_string conversion machinery
1 parent c0e6451 commit aa99ac2

5 files changed

Lines changed: 105 additions & 43 deletions

File tree

‎lib/inc/sys_string/config.h‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -92,12 +92,6 @@
9292
#define SYS_STRING_NO_UNIQUE_ADDRESS [[no_unique_address]]
9393
#endif
9494

95-
//GCC up to 11.3 has a weird constexpr bug in some palces
96-
#if __GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ > 2)
97-
#define SYS_STRING_BUGGY_CONSTEXPR constexpr
98-
#else
99-
#define SYS_STRING_BUGGY_CONSTEXPR
100-
#endif
10195

10296
#if __has_include(<version>)
10397
#include <version>

‎lib/inc/sys_string/impl/platforms/python_any.h‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,6 @@ namespace sysstr::util
4141
return src;
4242
}
4343

44-
template<size_t N>
45-
constexpr auto find_max_codepoint(const char32_t (&ar)[N]) -> char32_t
46-
{
47-
char32_t max = 0;
48-
for(char32_t c: ar) {
49-
if (c > max)
50-
max = c;
51-
}
52-
return max;
53-
}
54-
5544
class py_builder_storage
5645
{
5746
public:
@@ -539,24 +528,39 @@ namespace sysstr
539528

540529
namespace sysstr::util
541530
{
542-
template<util::ct_string U32Str, util::ct_string U16Str, util::ct_string U8Str>
531+
template<util::ct_string Str>
543532
inline auto make_static_sys_string_pystr() noexcept -> sys_string_pystr
544533
{
545-
constexpr ::size_t size = U32Str.size();
546-
constexpr auto maxChar = find_max_codepoint(U32Str.chars);
534+
constexpr auto maxChar = []() constexpr {
535+
utf_codepoint_decoder<utf8> decoder;
536+
char32_t max = 0;
537+
for (auto c: Str.chars)
538+
{
539+
decoder.put(c);
540+
if (decoder.error())
541+
throw "invalid UTF-8 constant";
542+
if (decoder.done()) {
543+
if (SYS_STRING_DECODER_VALUE(decoder) > max)
544+
max = SYS_STRING_DECODER_VALUE(decoder);
545+
}
546+
}
547+
return max;
548+
}();
547549
if constexpr (maxChar <= 0x7fu) {
548-
static PyUnicodeObject_wrapper<PyUnicode_1BYTE_KIND> str(size - 1, U8Str.chars);
550+
static PyUnicodeObject_wrapper<PyUnicode_1BYTE_KIND> str(Str.size() - 1, Str.chars);
549551
return str.as_string();
550552
} else if constexpr (maxChar <= 0xffffu) {
551-
static PyUnicodeObject_wrapper<PyUnicode_2BYTE_KIND> str(size - 1, U16Str.chars);
553+
static constexpr auto u16str = ct_utf8_to_utf16<Str>();
554+
static PyUnicodeObject_wrapper<PyUnicode_2BYTE_KIND> str(u16str.size() - 1, u16str.chars);
552555
return str.as_string();
553556
} else {
554-
static PyUnicodeObject_wrapper<PyUnicode_4BYTE_KIND> str(size - 1, U32Str.chars);
557+
static constexpr auto u32str = ct_utf8_to_utf32<Str>();
558+
static PyUnicodeObject_wrapper<PyUnicode_4BYTE_KIND> str(u32str.size() - 1, u32str.chars);
555559
return str.as_string();
556560
}
557561
}
558562
}
559563

560564

561-
#define SYS_STRING_STATIC_PYSTR(x) ::sysstr::util::make_static_sys_string_pystr<U##x, u##x, u8##x>()
565+
#define SYS_STRING_STATIC_PYSTR(x) ::sysstr::util::make_static_sys_string_pystr<u8##x>()
562566

‎lib/inc/sys_string/impl/unicode/mappings_common.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ namespace sysstr::util {
4646
decoder.put(*begin++);
4747
if (!decoder.done())
4848
decoder.put(*begin++); //no need to bounds check, we know end is good
49-
dest = write_unsafe<Enc>(decoder.value(), dest);
49+
dest = write_unsafe<Enc>(SYS_STRING_DECODER_VALUE(decoder), dest);
5050
}
5151
return dest;
5252
}

‎lib/inc/sys_string/impl/unicode/utf_encoding.h‎

Lines changed: 43 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ namespace sysstr
110110
constexpr bool error() const noexcept
111111
{ return m_state == state_error; }
112112

113-
SYS_STRING_BUGGY_CONSTEXPR uint32_t value() const noexcept
113+
constexpr uint32_t value() const noexcept
114114
{ return m_value; }
115115
private:
116116
static constexpr int state_done = 0;
@@ -125,7 +125,14 @@ namespace sysstr
125125
/* state_error */ 0, 4, 8, 0
126126
};
127127

128+
//GCC up to 11.3 has a weird constexpr bug in some places
129+
#if defined(__clang__) || !defined(__GNUC__) || (__GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ > 2))
130+
private:
131+
#else
132+
public:
133+
#endif
128134
uint32_t m_value = 0;
135+
private:
129136
int m_state = state_done;
130137
};
131138

@@ -153,7 +160,7 @@ namespace sysstr
153160
constexpr bool error() const noexcept
154161
{ return m_state == state_error; }
155162

156-
SYS_STRING_BUGGY_CONSTEXPR uint32_t value() const noexcept
163+
constexpr uint32_t value() const noexcept
157164
{ return m_value; }
158165

159166
private:
@@ -169,7 +176,14 @@ namespace sysstr
169176
/* state_error */ 0, 8, 4, 0
170177
};
171178

179+
//GCC up to 11.3 has a weird constexpr bug in some places
180+
#if defined(__clang__) || !defined(__GNUC__) || (__GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ > 2))
181+
private:
182+
#else
183+
public:
184+
#endif
172185
uint32_t m_value = 0;
186+
private:
173187
int m_state = state_done;
174188
};
175189

@@ -239,10 +253,16 @@ namespace sysstr
239253
constexpr bool error() const noexcept
240254
{ return m_state == state_reject; }
241255

242-
SYS_STRING_BUGGY_CONSTEXPR uint32_t value() const noexcept
256+
constexpr uint32_t value() const noexcept
243257
{ return m_value; }
258+
//GCC up to 11.3 has a weird constexpr bug in some places
259+
#if defined(__clang__) || !defined(__GNUC__) || (__GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ > 2))
244260
private:
261+
#else
262+
public:
263+
#endif
245264
uint32_t m_value = 0;
265+
private:
246266
uint8_t m_state = state_accept;
247267

248268

@@ -304,10 +324,16 @@ namespace sysstr
304324
constexpr bool error() const noexcept
305325
{ return m_state == state_reject; }
306326

307-
SYS_STRING_BUGGY_CONSTEXPR uint32_t value() const noexcept
327+
constexpr uint32_t value() const noexcept
308328
{ return m_value; }
329+
//GCC up to 11.3 has a weird constexpr bug in some places
330+
#if defined(__clang__) || !defined(__GNUC__) || (__GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ > 2))
309331
private:
332+
#else
333+
public:
334+
#endif
310335
uint32_t m_value = 0;
336+
private:
311337
uint8_t m_shift = 0;
312338
uint8_t m_state = state_accept;
313339

@@ -400,9 +426,15 @@ namespace sysstr
400426
return !( (c & 0xFFFFF800) == 0x0000D800 || c > 0x010FFFF );
401427
}
402428

403-
SYS_STRING_BUGGY_CONSTEXPR uint32_t value() const noexcept
429+
constexpr uint32_t value() const noexcept
404430
{ return m_value; }
431+
432+
//GCC up to 11.3 has a weird constexpr bug in some places
433+
#if defined(__clang__) || !defined(__GNUC__) || (__GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ > 2))
405434
private:
435+
#else
436+
public:
437+
#endif
406438
uint32_t m_value = 0;
407439
};
408440

@@ -431,6 +463,12 @@ namespace sysstr
431463
};
432464
}
433465

466+
#if defined(__clang__) || !defined(__GNUC__) || (__GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ > 2))
467+
#define SYS_STRING_DECODER_VALUE(dec) (dec).value()
468+
#else
469+
#define SYS_STRING_DECODER_VALUE(dec) (dec).m_value
470+
#endif
471+
434472
#endif
435473

436474

‎lib/inc/sys_string/impl/unicode/utf_util.h‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ namespace sysstr
179179
m_decoder.put(byte);
180180
if (m_decoder.done())
181181
{
182-
output(char32_t{m_decoder.value()});
182+
output(char32_t{SYS_STRING_DECODER_VALUE(m_decoder)});
183183
return;
184184
}
185185
if (m_decoder.error())
@@ -202,7 +202,7 @@ namespace sysstr
202202
m_decoder.put(uint16_t(c));
203203
if (m_decoder.done())
204204
{
205-
output(char32_t{m_decoder.value()});
205+
output(char32_t{SYS_STRING_DECODER_VALUE(m_decoder)});
206206
return;
207207
}
208208
if (m_decoder.error())
@@ -222,7 +222,7 @@ namespace sysstr
222222
void put(Char c) noexcept(noexcept(this->output(char32_t{}))) requires(From == utf32)
223223
{
224224
if (m_decoder.put(uint32_t(c)))
225-
output(char32_t{m_decoder.value()});
225+
output(char32_t{SYS_STRING_DECODER_VALUE(m_decoder)});
226226
else
227227
output(char32_t{U'\uFFFD'});
228228
}
@@ -284,7 +284,7 @@ namespace sysstr
284284
return char32_t{U'\uFFFD'};
285285
++first;
286286
if (decoder.done())
287-
return char32_t{decoder.value()};
287+
return char32_t{SYS_STRING_DECODER_VALUE(decoder)};
288288
if (first == last)
289289
return char32_t{U'\uFFFD'};
290290
}
@@ -296,7 +296,7 @@ namespace sysstr
296296
decoder.put(uint16_t(*first));
297297
++first;
298298
if (decoder.done())
299-
return char32_t{decoder.value()};
299+
return char32_t{SYS_STRING_DECODER_VALUE(decoder)};
300300

301301
if (decoder.error() || first == last)
302302
return char32_t{U'\uFFFD'};
@@ -306,14 +306,14 @@ namespace sysstr
306306
return char32_t{U'\uFFFD'};
307307
++first;
308308

309-
return char32_t{decoder.value()};
309+
return char32_t{SYS_STRING_DECODER_VALUE(decoder)};
310310
}
311311
else if constexpr (From == utf32)
312312
{
313313
utf_codepoint_decoder<utf32> decoder;
314314
bool res = decoder.put(uint32_t(*first));
315315
++first;
316-
return res ? char32_t{decoder.value()} : U'\uFFFD';
316+
return res ? char32_t{SYS_STRING_DECODER_VALUE(decoder)} : U'\uFFFD';
317317
}
318318

319319

@@ -346,7 +346,7 @@ namespace sysstr
346346
decoder.put(byte);
347347

348348
if (decoder.done())
349-
return char32_t{decoder.value()};
349+
return char32_t{SYS_STRING_DECODER_VALUE(decoder)};
350350

351351
if (decoder.error())
352352
{
@@ -368,7 +368,7 @@ namespace sysstr
368368
return char32_t{U'\uFFFD'};
369369
++first;
370370
if (decoder.done())
371-
return char32_t{decoder.value()};
371+
return char32_t{SYS_STRING_DECODER_VALUE(decoder)};
372372
if (first == last)
373373
return char32_t{U'\uFFFD'};
374374
}
@@ -381,7 +381,7 @@ namespace sysstr
381381
decoder.put(uint16_t(*first));
382382
++first;
383383
if (decoder.done())
384-
return char32_t{decoder.value()};
384+
return char32_t{SYS_STRING_DECODER_VALUE(decoder)};
385385

386386
if (decoder.error() || first == last)
387387
return char32_t{U'\uFFFD'};
@@ -391,14 +391,40 @@ namespace sysstr
391391
return char32_t{U'\uFFFD'};
392392
++first;
393393

394-
return char32_t{decoder.value()};
394+
return char32_t{SYS_STRING_DECODER_VALUE(decoder)};
395395
}
396396
else if constexpr (From == utf32)
397397
{
398398
return utf32_input::read(first, last);
399399
}
400400
}
401401

402+
template<util::ct_string Str>
403+
consteval auto ct_utf8_to_utf32() {
404+
constexpr size_t size = []() {
405+
utf_codepoint_decoder<utf8> decoder;
406+
size_t size = 0;
407+
for (auto c: Str.chars)
408+
{
409+
decoder.put(c);
410+
if (decoder.error())
411+
throw "invalid UTF-8 constant";
412+
size += decoder.done();
413+
}
414+
return size;
415+
}();
416+
utf_codepoint_decoder<utf8> decoder;
417+
char32_t ret[size];
418+
size_t idx = 0;
419+
for (auto c: Str.chars)
420+
{
421+
decoder.put(c);
422+
if (decoder.done())
423+
ret[idx++] = SYS_STRING_DECODER_VALUE(decoder);
424+
}
425+
return util::ct_string<char32_t, size>(ret);
426+
}
427+
402428
template<util::ct_string Str>
403429
consteval auto ct_utf8_to_utf16() {
404430
constexpr size_t size = []() {
@@ -412,7 +438,7 @@ namespace sysstr
412438
throw "invalid UTF-8 constant";
413439
if (decoder.done())
414440
{
415-
encoder.put(decoder.value());
441+
encoder.put(SYS_STRING_DECODER_VALUE(decoder));
416442
size += encoder.size();
417443
}
418444
}
@@ -427,7 +453,7 @@ namespace sysstr
427453
decoder.put(c);
428454
if (decoder.done())
429455
{
430-
encoder.put(decoder.value());
456+
encoder.put(SYS_STRING_DECODER_VALUE(decoder));
431457
for(char16_t uc: encoder)
432458
ret[idx++] = uc;
433459
}

0 commit comments

Comments
 (0)