ObjFW  Check-in [c51761c6e9]

Overview
Comment:Split Unicode table generator and download CaseFolding.txt.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: c51761c6e9d9c38fcbae95c4fa6daac48526c1f0f853293e1ba416962b67f67f
User & Date: js on 2009-10-29 14:16:10
Other Links: manifest | tags
Context
2009-10-31
14:46
Add case folding table generator. check-in: a95f375c28 user: js tags: trunk
2009-10-29
14:16
Split Unicode table generator and download CaseFolding.txt. check-in: c51761c6e9 user: js tags: trunk
2009-10-19
22:09
Automatically download UnicodeData.txt if it's missing. check-in: c5d489cefd user: js tags: trunk
Changes

Modified generators/Makefile from [3e90090f60] to [7ae395816f].

1
2

3
4
5
6
7
8
9
10
11
12
PROG_NOINST = gen_tables${PROG_SUFFIX}
SRCS = gen_tables.m


.PHONY: run
run: all UnicodeData.txt
	rm -f libobjfw.so.0 libobjfw.so.0.1 libobjfw.dll libobjfw.dylib
	if test -f ../src/libobjfw.so; then \
		ln -s ../src/libobjfw.so libobjfw.so.0; \
		ln -s ../src/libobjfw.so libobjfw.so.0.1; \
	elif test -f ../src/libobjfw.so.0.1; then \
		ln -s ../src/libobjfw.so.0.1 libobjfw.so.0.1; \
	fi

|
>


|







1
2
3
4
5
6
7
8
9
10
11
12
13
PROG_NOINST = gen_tables${PROG_SUFFIX}
SRCS = main.o			\
       UpperLowerGenerator.m

.PHONY: run
run: all UnicodeData.txt CaseFolding.txt
	rm -f libobjfw.so.0 libobjfw.so.0.1 libobjfw.dll libobjfw.dylib
	if test -f ../src/libobjfw.so; then \
		ln -s ../src/libobjfw.so libobjfw.so.0; \
		ln -s ../src/libobjfw.so libobjfw.so.0.1; \
	elif test -f ../src/libobjfw.so.0.1; then \
		ln -s ../src/libobjfw.so.0.1 libobjfw.so.0.1; \
	fi
22
23
24
25
26
27
28



29
30
31
32
	${TEST_LAUNCHER} ./${PROG_NOINST}; EXIT=$$?; \
	rm -f libobjfw.so.0 libobjfw.so.0.1 libobjfw.dll libobjfw.dylib; \
	exit $$EXIT

UnicodeData.txt:
	wget http://unicode.org/Public/UNIDATA/UnicodeData.txt




include ../buildsys.mk

CPPFLAGS += -I../src -I..
LIBS := -L../src -lobjfw ${LIBS}







>
>
>




23
24
25
26
27
28
29
30
31
32
33
34
35
36
	${TEST_LAUNCHER} ./${PROG_NOINST}; EXIT=$$?; \
	rm -f libobjfw.so.0 libobjfw.so.0.1 libobjfw.dll libobjfw.dylib; \
	exit $$EXIT

UnicodeData.txt:
	wget http://unicode.org/Public/UNIDATA/UnicodeData.txt

CaseFolding.txt:
	wget http://www.unicode.org/Public/UNIDATA/CaseFolding.txt

include ../buildsys.mk

CPPFLAGS += -I../src -I..
LIBS := -L../src -lobjfw ${LIBS}

Added generators/UpperLowerGenerator.h version [4a2f69fdd1].



























































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
/*
 * Copyright (c) 2008 - 2009
 *   Jonathan Schleifer <js@webkeks.org>
 *
 * All rights reserved.
 *
 * This file is part of libobjfw. It may be distributed under the terms of the
 * Q Public License 1.0, which can be found in the file LICENSE included in
 * the packaging of this file.
 */

#import "OFString.h"

@interface UpperLowerGenerator: OFObject
{
	of_unichar_t upper[0x110000];
	of_unichar_t lower[0x110000];
}

- (void)fillTablesFromFile: (OFString*)file;
- (size_t)writeTable: (of_unichar_t*)table
	    withName: (OFString*)name
	      toFile: (OFString*)file;
- (size_t)writeUpperTableToFile: (OFString*)file;
- (size_t)writeLowerTableToFile: (OFString*)file;
- (void)writeHeaderToFile: (OFString*)file
       withUpperTableSize: (size_t)upper_size
	   lowerTableSize: (size_t)lower_size;
@end

Added generators/UpperLowerGenerator.m version [cb98ef8d98].





































































































































































































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
/*
 * Copyright (c) 2008 - 2009
 *   Jonathan Schleifer <js@webkeks.org>
 *
 * All rights reserved.
 *
 * This file is part of libobjfw. It may be distributed under the terms of the
 * Q Public License 1.0, which can be found in the file LICENSE included in
 * the packaging of this file.
 */

#include "config.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#import "OFFile.h"
#import "OFAutoreleasePool.h"

#import "UpperLowerGenerator.h"
#import "copyright.h"

@implementation UpperLowerGenerator
- (void)fillTablesFromFile: (OFString*)file;
{
	OFAutoreleasePool *pool = [[OFAutoreleasePool alloc] init], *pool2;
	OFFile *src = [OFFile fileWithPath: file
				      mode: @"rb"];
	OFString *line;

	pool2 = [[OFAutoreleasePool alloc] init];
	while ((line = [src readLine])) {
		OFArray *splitted;
		OFString **splitted_carray;
		of_unichar_t codep;

		splitted = [line splitWithDelimiter: @";"];
		if ([splitted count] != 15) {
			fprintf(stderr, "Invalid line: %s\n", [line cString]);
			exit(1);
		}
		splitted_carray = [splitted cArray];

		codep = [splitted_carray[0] hexadecimalValueAsInteger];
		upper[codep] = [splitted_carray[12] hexadecimalValueAsInteger];
		lower[codep] = [splitted_carray[13] hexadecimalValueAsInteger];

		[pool2 releaseObjects];
	}

	[pool release];
}

- (size_t)writeTable: (of_unichar_t*)table
	    withName: (OFString*)name
	      toFile: (OFString*)file
{
	OFAutoreleasePool *pool = [[OFAutoreleasePool alloc] init];
	OFAutoreleasePool *pool2;
	OFFile *f = [OFFile fileWithPath: file
				    mode: @"wb"];

	of_unichar_t i, j;
	size_t last_used = SIZE_MAX;
	BOOL table_used[0x1100];

	memset(table_used, NO, 0x1100);

	[f writeString: COPYRIGHT
	    @"#include \"config.h\"\n"
	    @"\n"
	    @"#import \"OFString.h\"\n\n"];

	[f writeString: @"static const of_unichar_t nop_page[0x100] = {};\n\n"];

	for (i = 0; i < 0x110000; i += 0x100) {
		BOOL empty;

		empty = YES;

		for (j = i; j < i + 0x100; j++) {
			if (table[j] != 0) {
				empty = NO;
				last_used = i >> 8;
				table_used[last_used] = YES;
			}
		}

		if (!empty) {
		       	pool2 = [[OFAutoreleasePool alloc] init];

			[f writeString: [OFString stringWithFormat:
			    @"static const of_unichar_t page_%d[0x100] = {\n",
			    i >> 8]];

			for (j = i; j < i + 0x100; j += 4) {
				[f writeString: [OFString stringWithFormat:
				    @"\t0x%06X, 0x%06X, 0x%06X, 0x%06X,\n",
				    table[j], table[j + 1], table[j + 2],
				    table[j + 3]]];

				[pool2 releaseObjects];
			}

			[f writeString: @"};\n\n"];

			[pool2 release];
		}
	}

	last_used++;

	[f writeString: [OFString stringWithFormat:
	    @"const of_unichar_t* const of_unicode_%s_table[0x%X] = {\n\t",
	    [name cString], last_used]];

	pool2 = [[OFAutoreleasePool alloc] init];
	for (i = 0; i < last_used; i++) {
		if (table_used[i])
			[f writeString: [OFString stringWithFormat: @"page_%d",
								    i]];
		else
			[f writeString: @"nop_page"];

		if ((i + 1) % 4 == 0)
			[f writeString: @",\n\t"];
		else if (i + 1 < last_used)
			[f writeString: @", "];

		[pool2 releaseObjects];
	}

	[f writeString: @"\n};\n"];

	[pool release];
	return last_used;
}

- (size_t)writeUpperTableToFile: (OFString*)file
{
	return [self writeTable: upper
		       withName: @"upper"
			 toFile: file];
}

- (size_t)writeLowerTableToFile: (OFString*)file
{
	return [self writeTable: lower
		       withName: @"lower"
			 toFile: file];
}

- (void)writeHeaderToFile: (OFString*)file
       withUpperTableSize: (size_t)upper_size
	   lowerTableSize: (size_t)lower_size
{
	OFAutoreleasePool *pool = [[OFAutoreleasePool alloc] init];
	OFFile *f = [OFFile fileWithPath: file
				    mode: @"wb"];

	[f writeString: COPYRIGHT
	    @"#import \"OFString.h\"\n\n"];

	[f writeString: [OFString stringWithFormat:
	    @"#define OF_UNICODE_UPPER_TABLE_SIZE 0x%X\n"
	    @"#define OF_UNICODE_LOWER_TABLE_SIZE 0x%X\n\n",
	    upper_size, lower_size]];

	[f writeString:
	    @"extern const of_unichar_t* const\n"
	    @"    of_unicode_upper_table[OF_UNICODE_UPPER_TABLE_SIZE];\n"
	    @"extern const of_unichar_t* const\n"
	    @"    of_unicode_lower_table[OF_UNICODE_LOWER_TABLE_SIZE];\n"];

	[pool release];
}
@end

Added generators/copyright.h version [bcace0fd16].























































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
/*
 * Copyright (c) 2008 - 2009
 *   Jonathan Schleifer <js@webkeks.org>
 *
 * All rights reserved.
 *
 * This file is part of libobjfw. It may be distributed under the terms of the
 * Q Public License 1.0, which can be found in the file LICENSE included in
 * the packaging of this file.
 */

#import "OFString.h"

#define COPYRIGHT \
    @"/*\n" \
    @" * Copyright (c) 2008 - 2009\n" \
    @" *   Jonathan Schleifer <js@webkeks.org>\n" \
    @" *\n" \
    @" * All rights reserved.\n" \
    @" *\n" \
    @" * This file is part of libobjfw. It may be distributed under the " \
    @"terms of the\n" \
    @" * Q Public License 1.0, which can be found in the file LICENSE " \
    @"included in\n" \
    @" * the packaging of this file.\n" \
    @" */\n" \
    @"\n"

Deleted generators/gen_tables.m version [8540a1b692].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
/*
 * Copyright (c) 2008 - 2009
 *   Jonathan Schleifer <js@webkeks.org>
 *
 * All rights reserved.
 *
 * This file is part of libobjfw. It may be distributed under the terms of the
 * Q Public License 1.0, which can be found in the file LICENSE included in
 * the packaging of this file.
 */

#include "config.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#import "OFString.h"
#import "OFFile.h"
#import "OFAutoreleasePool.h"

#define COPYRIGHT \
    @"/*\n" \
    @" * Copyright (c) 2008 - 2009\n" \
    @" *   Jonathan Schleifer <js@webkeks.org>\n" \
    @" *\n" \
    @" * All rights reserved.\n" \
    @" *\n" \
    @" * This file is part of libobjfw. It may be distributed under the " \
    @"terms of the\n" \
    @" * Q Public License 1.0, which can be found in the file LICENSE " \
    @"included in\n" \
    @" * the packaging of this file.\n" \
    @" */\n" \
    @"\n"

@interface TableGenerator: OFObject
{
	of_unichar_t upper[0x110000];
	of_unichar_t lower[0x110000];
}

- (void)fillTablesFromFile: (OFString*)file;
- (size_t)writeTable: (of_unichar_t*)table
	    withName: (OFString*)name
	      toFile: (OFString*)file;
- (size_t)writeUpperTableToFile: (OFString*)file;
- (size_t)writeLowerTableToFile: (OFString*)file;
- (void)writeHeaderToFile: (OFString*)file
       withUpperTableSize: (size_t)upper_size
	   lowerTableSize: (size_t)lower_size;
@end

@implementation TableGenerator
- (void)fillTablesFromFile: (OFString*)file;
{
	OFAutoreleasePool *pool = [[OFAutoreleasePool alloc] init], *pool2;
	OFFile *src = [OFFile fileWithPath: file
				      mode: @"rb"];
	OFString *line;

	pool2 = [[OFAutoreleasePool alloc] init];
	while ((line = [src readLine])) {
		OFArray *splitted;
		OFString **splitted_carray;
		of_unichar_t codep;

		splitted = [line splitWithDelimiter: @";"];
		if ([splitted count] != 15) {
			fprintf(stderr, "Invalid line: %s\n", [line cString]);
			exit(1);
		}
		splitted_carray = [splitted cArray];

		codep = [splitted_carray[0] hexadecimalValueAsInteger];
		upper[codep] = [splitted_carray[12] hexadecimalValueAsInteger];
		lower[codep] = [splitted_carray[13] hexadecimalValueAsInteger];

		[pool2 releaseObjects];
	}

	[pool release];
}

- (size_t)writeTable: (of_unichar_t*)table
	    withName: (OFString*)name
	      toFile: (OFString*)file
{
	OFAutoreleasePool *pool = [[OFAutoreleasePool alloc] init];
	OFAutoreleasePool *pool2;
	OFFile *f = [OFFile fileWithPath: file
				    mode: @"wb"];

	of_unichar_t i, j;
	size_t last_used = SIZE_MAX;
	BOOL table_used[0x1100];

	memset(table_used, NO, 0x1100);

	[f writeString: COPYRIGHT
	    @"#include \"config.h\"\n"
	    @"\n"
	    @"#import \"OFString.h\"\n\n"];

	[f writeString: @"static const of_unichar_t nop_page[0x100] = {};\n\n"];

	for (i = 0; i < 0x110000; i += 0x100) {
		BOOL empty;

		empty = YES;

		for (j = i; j < i + 0x100; j++) {
			if (table[j] != 0) {
				empty = NO;
				last_used = i >> 8;
				table_used[last_used] = YES;
			}
		}

		if (!empty) {
		       	pool2 = [[OFAutoreleasePool alloc] init];

			[f writeString: [OFString stringWithFormat:
			    @"static const of_unichar_t page_%d[0x100] = {\n",
			    i >> 8]];

			for (j = i; j < i + 0x100; j += 4) {
				[f writeString: [OFString stringWithFormat:
				    @"\t0x%06X, 0x%06X, 0x%06X, 0x%06X,\n",
				    table[j], table[j + 1], table[j + 2],
				    table[j + 3]]];

				[pool2 releaseObjects];
			}

			[f writeString: @"};\n\n"];

			[pool2 release];
		}
	}

	last_used++;

	[f writeString: [OFString stringWithFormat:
	    @"const of_unichar_t* const of_unicode_%s_table[0x%X] = {\n\t",
	    [name cString], last_used]];

	pool2 = [[OFAutoreleasePool alloc] init];
	for (i = 0; i < last_used; i++) {
		if (table_used[i])
			[f writeString: [OFString stringWithFormat: @"page_%d",
								    i]];
		else
			[f writeString: @"nop_page"];

		if ((i + 1) % 4 == 0)
			[f writeString: @",\n\t"];
		else if (i + 1 < last_used)
			[f writeString: @", "];

		[pool2 releaseObjects];
	}

	[f writeString: @"\n};\n"];

	[pool release];
	return last_used;
}

- (size_t)writeUpperTableToFile: (OFString*)file
{
	return [self writeTable: upper
		       withName: @"upper"
			 toFile: file];
}

- (size_t)writeLowerTableToFile: (OFString*)file
{
	return [self writeTable: lower
		       withName: @"lower"
			 toFile: file];
}

- (void)writeHeaderToFile: (OFString*)file
       withUpperTableSize: (size_t)upper_size
	   lowerTableSize: (size_t)lower_size
{
	OFAutoreleasePool *pool = [[OFAutoreleasePool alloc] init];
	OFFile *f = [OFFile fileWithPath: file
				    mode: @"wb"];

	[f writeString: COPYRIGHT
	    @"#import \"OFString.h\"\n\n"];

	[f writeString: [OFString stringWithFormat:
	    @"#define OF_UNICODE_UPPER_TABLE_SIZE 0x%X\n"
	    @"#define OF_UNICODE_LOWER_TABLE_SIZE 0x%X\n\n",
	    upper_size, lower_size]];

	[f writeString:
	    @"extern const of_unichar_t* const\n"
	    @"    of_unicode_upper_table[OF_UNICODE_UPPER_TABLE_SIZE];\n"
	    @"extern const of_unichar_t* const\n"
	    @"    of_unicode_lower_table[OF_UNICODE_LOWER_TABLE_SIZE];\n"];

	[pool release];
}
@end

int
main()
{
	OFAutoreleasePool *pool = [[OFAutoreleasePool alloc] init];
	TableGenerator *tgen = [[[TableGenerator alloc] init] autorelease];
	size_t upper_size, lower_size;

	[tgen fillTablesFromFile: @"UnicodeData.txt"];
	upper_size = [tgen writeUpperTableToFile: @"../src/unicode_upper.m"];
	lower_size = [tgen writeLowerTableToFile: @"../src/unicode_lower.m"];
	[tgen writeHeaderToFile: @"../src/unicode.h"
	     withUpperTableSize: upper_size
		 lowerTableSize: lower_size];

	[pool release];

	return 0;
}
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<






































































































































































































































































































































































































































































Added generators/main.m version [50bb57fa45].









































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
/*
 * Copyright (c) 2008 - 2009
 *   Jonathan Schleifer <js@webkeks.org>
 *
 * All rights reserved.
 *
 * This file is part of libobjfw. It may be distributed under the terms of the
 * Q Public License 1.0, which can be found in the file LICENSE included in
 * the packaging of this file.
 */

#include "config.h"

#import "OFAutoreleasePool.h"

#import "UpperLowerGenerator.h"

int
main()
{
	OFAutoreleasePool *pool = [[OFAutoreleasePool alloc] init];
	UpperLowerGenerator *tgen;
	size_t upper_size, lower_size;

	tgen = [[[UpperLowerGenerator alloc] init] autorelease];
	[tgen fillTablesFromFile: @"UnicodeData.txt"];
	upper_size = [tgen writeUpperTableToFile: @"../src/unicode_upper.m"];
	lower_size = [tgen writeLowerTableToFile: @"../src/unicode_lower.m"];
	[tgen writeHeaderToFile: @"../src/unicode.h"
	     withUpperTableSize: upper_size
		 lowerTableSize: lower_size];

	[pool release];

	return 0;
}