ObjFW  Check-in [edb216ca88]

Overview
Comment:More efficient Windows-1252 -> UTF-8 conversion
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: edb216ca8839803bd83d6752cacbbfe3a8d3640edfcc417760700cab6697ba97
User & Date: js on 2017-02-26 14:57:55
Other Links: manifest | tags
Context
2017-02-26
15:34
Update README.md check-in: e8b1d00205 user: js tags: trunk
14:57
More efficient Windows-1252 -> UTF-8 conversion check-in: edb216ca88 user: js tags: trunk
14:30
More efficient Windows-1251 -> UTF-8 conversion check-in: 928ee025eb user: js tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Modified src/encodings/windows_1252.m from [0b9f3c6c61] to [bd2b6d5026].

︙︙
13
14
15
16
17
18
19


20
21
22
23
24
25
26

 * LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this
 * file.
 */

#include "config.h"

#import "OFString.h"



const of_char16_t of_windows_1252_table[] = {
	0x20AC, 0xFFFF, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
	0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFF, 0x017D, 0xFFFF,
	0xFFFF, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
	0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFF, 0x017E, 0x0178,
	0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,








>
>








13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

 * LICENSE.GPLv2 or LICENSE.GPLv3 respectively included in the packaging of this
 * file.
 */

#include "config.h"

#import "OFString.h"

#import "common.h"

const of_char16_t of_windows_1252_table[] = {
	0x20AC, 0xFFFF, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
	0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0xFFFF, 0x017D, 0xFFFF,
	0xFFFF, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
	0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0xFFFF, 0x017E, 0x0178,
	0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7,
︙︙
35
36
37
38
39
40
41

























































42
43
44
45
46
47
48
49


50
51
52
53
54
55
56
57
58
59
60
61


62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160

	0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
	0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
	0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF
};
const size_t of_windows_1252_table_offset =
    256 - (sizeof(of_windows_1252_table) / sizeof(*of_windows_1252_table));


























































bool
of_unicode_to_windows_1252(const of_unichar_t *input, unsigned char *output,
    size_t length, bool lossy)
{
	for (size_t i = 0; i < length; i++) {
		of_unichar_t c = input[i];

		if OF_UNLIKELY (c > 0xFF) {


			if OF_UNLIKELY (c > 0xFFFF) {
				if (lossy) {
					output[i] = '?';
					continue;
				} else
					return false;
			}

			switch ((of_char16_t)c) {
			case 0x20AC:
				output[i] = 0x80;
				break;


			case 0x201A:
				output[i] = 0x82;
				break;
			case 0x192:
				output[i] = 0x83;
				break;
			case 0x201E:
				output[i] = 0x84;
				break;
			case 0x2026:
				output[i] = 0x85;
				break;
			case 0x2020:
				output[i] = 0x86;
				break;
			case 0x2021:
				output[i] = 0x87;
				break;
			case 0x2C6:
				output[i] = 0x88;
				break;
			case 0x2030:
				output[i] = 0x89;
				break;
			case 0x160:
				output[i] = 0x8A;
				break;
			case 0x2039:
				output[i] = 0x8B;
				break;
			case 0x152:
				output[i] = 0x8C;
				break;
			case 0x17D:
				output[i] = 0x8E;
				break;
			case 0x2018:
				output[i] = 0x91;
				break;
			case 0x2019:
				output[i] = 0x92;
				break;
			case 0x201C:
				output[i] = 0x93;
				break;
			case 0x201D:
				output[i] = 0x94;
				break;
			case 0x2022:
				output[i] = 0x95;
				break;
			case 0x2013:
				output[i] = 0x96;
				break;
			case 0x2014:
				output[i] = 0x97;
				break;
			case 0x2DC:
				output[i] = 0x98;
				break;
			case 0x2122:
				output[i] = 0x99;
				break;
			case 0x161:
				output[i] = 0x9A;
				break;
			case 0x203A:
				output[i] = 0x9B;
				break;
			case 0x153:
				output[i] = 0x9C;
				break;
			case 0x17E:
				output[i] = 0x9E;
				break;
			case 0x178:
				output[i] = 0x9F;
				break;
			default:
				if (lossy)
					output[i] = '?';

				else
					return false;

				break;
			}
		} else {
			if OF_UNLIKELY (c >= 0x80 && c <= 0x9F) {
				if (lossy)
					output[i] = '?';
				else
					return false;
			} else
				output[i] = (unsigned char)c;
		}
	}

	return true;
}








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







|
>
>








|
|
<
<
>
>
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
|
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<

|

>
|

|
<
<
<
<
<
<
<
<
|
|
<





37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120


121
122
123














124






























































125
126
127
128
129
130
131








132
133

134
135
136
137

	0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
	0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7,
	0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF
};
const size_t of_windows_1252_table_offset =
    256 - (sizeof(of_windows_1252_table) / sizeof(*of_windows_1252_table));

static const unsigned char page0[] = {
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
static const uint8_t page0Start = 0x80;

static const unsigned char page1[] = {
	0x8C, 0x9C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x9A,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x00,
	0x00, 0x00, 0x00, 0x8E, 0x9E, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x83
};
static const uint8_t page1Start = 0x52;

static const unsigned char page2[] = {
	0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98
};
static const uint8_t page2Start = 0xC6;

static const unsigned char page20[] = {
	0x96, 0x97, 0x00, 0x00, 0x00, 0x91, 0x92, 0x82,
	0x00, 0x93, 0x94, 0x84, 0x00, 0x86, 0x87, 0x95,
	0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x9B,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
	0x00, 0x80
};
static const uint8_t page20Start = 0x13;

static const unsigned char page21[] = {
	0x99
};
static const uint8_t page21Start = 0x22;

bool
of_unicode_to_windows_1252(const of_unichar_t *input, unsigned char *output,
    size_t length, bool lossy)
{
	for (size_t i = 0; i < length; i++) {
		of_unichar_t c = input[i];

		if OF_UNLIKELY (c > 0x7F) {
			uint8_t index;

			if OF_UNLIKELY (c > 0xFFFF) {
				if (lossy) {
					output[i] = '?';
					continue;
				} else
					return false;
			}

			switch (c >> 8) {
			CASE_MISSING_IS_KEEP(0)


			CASE_MISSING_IS_ERROR(1)
			CASE_MISSING_IS_ERROR(2)
			CASE_MISSING_IS_ERROR(20)














			CASE_MISSING_IS_ERROR(21)






























































			default:
				if (lossy) {
					output[i] = '?';
					continue;
				} else
					return false;
			}








		} else
			output[i] = (unsigned char)c;

	}

	return true;
}