Overview
| Comment: | More efficient Windows-1251 -> UTF-8 conversion |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA3-256: |
928ee025eb9cedcfb18f30d292b9ba86 |
| User & Date: | js on 2017-02-26 14:30:11 |
| Other Links: | manifest | tags |
Context
|
2017-02-26
| ||
| 14:57 | More efficient Windows-1252 -> UTF-8 conversion check-in: edb216ca88 user: js tags: trunk | |
| 14:30 | More efficient Windows-1251 -> UTF-8 conversion check-in: 928ee025eb user: js tags: trunk | |
| 12:45 | More efficient Mac Roman -> UTF-8 conversion check-in: 230644c87b user: js tags: trunk | |
Changes
Modified src/encodings/windows_1251.m from [157cfc4141] to [5235c111dd].
| ︙ | ︙ | |||
13 14 15 16 17 18 19 20 21 22 23 24 25 26 |
* LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this
* file.
*/
#include "config.h"
#import "OFString.h"
const of_char16_t of_windows_1251_table[] = {
0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021,
0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F,
0x0452, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0xFFFF, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F,
0x00A0, 0x040E, 0x045E, 0x0408, 0x00A4, 0x0490, 0x00A6, 0x00A7,
| > > | 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 |
* LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this
* file.
*/
#include "config.h"
#import "OFString.h"
#import "common.h"
const of_char16_t of_windows_1251_table[] = {
0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021,
0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F,
0x0452, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
0xFFFF, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F,
0x00A0, 0x040E, 0x045E, 0x0408, 0x00A4, 0x0490, 0x00A6, 0x00A7,
|
| ︙ | ︙ | |||
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447,
0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F
};
const size_t of_windows_1251_table_offset =
256 - (sizeof(of_windows_1251_table) / sizeof(*of_windows_1251_table));
bool
of_unicode_to_windows_1251(const of_unichar_t *input, unsigned char *output,
size_t length, bool lossy)
{
for (size_t i = 0; i < length; i++) {
of_unichar_t c = input[i];
if OF_UNLIKELY (c > 0x7F) {
if OF_UNLIKELY (c > 0xFFFF) {
if (lossy) {
output[i] = '?';
continue;
} else
return false;
}
| > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < < < | < < < < < < > > | < < < < < < < < < < < < < < | < | < > | | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | < < < | 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 |
0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F,
0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447,
0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F
};
const size_t of_windows_1251_table_offset =
256 - (sizeof(of_windows_1251_table) / sizeof(*of_windows_1251_table));
static const unsigned char page0[] = {
0xA0, 0x00, 0x00, 0x00, 0xA4, 0x00, 0xA6, 0xA7,
0x00, 0xA9, 0x00, 0xAB, 0xAC, 0xAD, 0xAE, 0x00,
0xB0, 0xB1, 0x00, 0x00, 0x00, 0xB5, 0xB6, 0xB7,
0x00, 0x00, 0x00, 0xBB
};
static const uint8_t page0Start = 0xA0;
static const unsigned char page4[] = {
0xA8, 0x80, 0x81, 0xAA, 0xBD, 0xB2, 0xAF, 0xA3,
0x8A, 0x8C, 0x8E, 0x8D, 0x00, 0xA1, 0x8F, 0xC0,
0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8,
0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xD0,
0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8,
0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0,
0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8,
0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xF0,
0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8,
0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x00,
0xB8, 0x90, 0x83, 0xBA, 0xBE, 0xB3, 0xBF, 0xBC,
0x9A, 0x9C, 0x9E, 0x9D, 0x00, 0xA2, 0x9F, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA5,
0xB4
};
static const uint8_t page4Start = 0x01;
static const unsigned char page20[] = {
0x96, 0x97, 0x00, 0x00, 0x00, 0x91, 0x92, 0x82,
0x00, 0x93, 0x94, 0x84, 0x00, 0x86, 0x87, 0x95,
0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x9B,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x88
};
static const uint8_t page20Start = 0x13;
static const unsigned char page21[] = {
0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x99
};
static const uint8_t page21Start = 0x16;
bool
of_unicode_to_windows_1251(const of_unichar_t *input, unsigned char *output,
size_t length, bool lossy)
{
for (size_t i = 0; i < length; i++) {
of_unichar_t c = input[i];
if OF_UNLIKELY (c > 0x7F) {
uint8_t index;
if OF_UNLIKELY (c > 0xFFFF) {
if (lossy) {
output[i] = '?';
continue;
} else
return false;
}
switch (c >> 8) {
CASE_MISSING_IS_ERROR(0)
CASE_MISSING_IS_ERROR(4)
CASE_MISSING_IS_ERROR(20)
CASE_MISSING_IS_ERROR(21)
default:
if (lossy) {
output[i] = '?';
continue;
} else
return false;
}
} else
output[i] = (unsigned char)c;
}
return true;
}
|