ObjFW  Check-in [516517deb3]

Overview
Comment:OFURL: Do not URL decode and reencode parts

URL decoding and reencoding is not lossless: For example, if the query
was foo=bar&qux=foo%25bar, it will be decoded to foo=bar&qux=foo&bar and
then reencoded to foo=bar%25qux=foo%25bar, which is a different thing.

The only way to solve this is to let the application handle the URL
decoding and encoding according to its own rules, as those might be
different depending on the application.

Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 516517deb3b6c5ff7c2e51dd44cacb432b0b1e081d918ed0d72107ca77960431
User & Date: js on 2016-08-21 14:00:20
Other Links: manifest | tags
Context
2016-08-21
14:09
OFHTTPClient: Add response to redirect delegate check-in: a509ab7e91 user: js tags: trunk
14:00
OFURL: Do not URL decode and reencode parts check-in: 516517deb3 user: js tags: trunk
2016-08-15
00:07
Support SjLj C++ EH on Darwin with ObjFW runtime check-in: 894a87f823 user: js tags: trunk
Changes

Modified src/OFHTTPClient.m from [01bffed1c2] to [5d3409efa0].

314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
314
315
316
317
318
319
320

321
322
323
324
325
326
327







-







- (OFHTTPResponse*)performRequest: (OFHTTPRequest*)request
			redirects: (size_t)redirects
{
	void *pool = objc_autoreleasePoolPush();
	OFURL *URL = [request URL];
	OFString *scheme = [URL scheme];
	of_http_request_method_t method = [request method];
	OFString *path;
	OFMutableString *requestString;
	OFString *user, *password;
	OFMutableDictionary OF_GENERIC(OFString*, OFString*) *headers;
	OFDataArray *body = [request body];
	OFTCPSocket *socket;
	OFHTTPClientResponse *response;
	OFString *line, *version, *redirect, *connectionHeader;
371
372
373
374
375
376
377
378
379
380
381
382
383


384
385
386
387
388
389


390
391
392
393
394
395
396
370
371
372
373
374
375
376

377
378
379


380
381
382
383
384
385


386
387
388
389
390
391
392
393
394







-



-
-
+
+




-
-
+
+







		socket = [self OF_closeAndCreateSocketForRequest: request];

	/*
	 * As a work around for a bug with split packets in lighttpd when using
	 * HTTPS, we construct the complete request in a buffer string and then
	 * send it all at once.
	 */
	path = [[URL path] stringByURLEncodingWithIgnoredCharacters: "/"];

	if ([URL query] != nil)
		requestString = [OFMutableString stringWithFormat:
		    @"%s /%@?%@ HTTP/%@\r\n",
		    of_http_request_method_to_string(method), path,
		    @"%s %@?%@ HTTP/%@\r\n",
		    of_http_request_method_to_string(method), [URL path],
		    [[URL query] stringByURLEncoding],
		    [request protocolVersionString]];
	else
		requestString = [OFMutableString stringWithFormat:
		    @"%s /%@ HTTP/%@\r\n",
		    of_http_request_method_to_string(method), path,
		    @"%s %@ HTTP/%@\r\n",
		    of_http_request_method_to_string(method), [URL path],
		    [request protocolVersionString]];

	headers = [[[request headers] mutableCopy] autorelease];
	if (headers == nil)
		headers = [OFMutableDictionary dictionary];

	if ([headers objectForKey: @"Host"] == nil) {

Modified src/OFString+URLEncoding.h from [fc4768ddc2] to [9f42522241].

34
35
36
37
38
39
40
41

42
43
44
45

46
47
48
49
50
51
52
53
54
55
34
35
36
37
38
39
40

41
42
43
44

45
46
47
48
49
50
51
52
53
54
55







-
+



-
+










 */
- (OFString*)stringByURLEncoding;

/*!
 * @brief Encodes a string for use in a URL, but does not escape the specified
 *	  ignored characters.
 *
 * @param ignored A C string of characters that should not be escaped
 * @param allowed A C string of characters that should not be escaped
 *
 * @return A new autoreleased string
 */
- (OFString*)stringByURLEncodingWithIgnoredCharacters: (const char*)ignored;
- (OFString*)stringByURLEncodingWithAllowedCharacters: (const char*)allowed;

/*!
 * @brief Decodes a string used in a URL.
 *
 * @return A new autoreleased string
 */
- (OFString*)stringByURLDecoding;
@end

OF_ASSUME_NONNULL_END

Modified src/OFString+URLEncoding.m from [134edb9cf7] to [9b57fed9b2].

27
28
29
30
31
32
33
34

35
36
37

38
39
40
41
42
43
44
27
28
29
30
31
32
33

34
35
36

37
38
39
40
41
42
43
44







-
+


-
+








/* Reference for static linking */
int _OFString_URLEncoding_reference;

@implementation OFString (URLEncoding)
- (OFString*)stringByURLEncoding
{
	return [self stringByURLEncodingWithIgnoredCharacters: ""];
	return [self stringByURLEncodingWithAllowedCharacters: "$-_.!*()"];
}

- (OFString*)stringByURLEncodingWithIgnoredCharacters: (const char*)ignored
- (OFString*)stringByURLEncodingWithAllowedCharacters: (const char*)allowed
{
	void *pool = objc_autoreleasePoolPush();
	const char *string = [self UTF8String];
	char *retCString;
	size_t i;
	OFString *ret;

55
56
57
58
59
60
61
62

63
64
65
66
67
68
69
70
71
55
56
57
58
59
60
61

62


63
64
65
66
67
68
69







-
+
-
-







		unsigned char c = *string;

		/*
		 * '+' is also listed in RFC 1738, however, '+' is sometimes
		 * interpreted as space in HTTP. Therefore always escape it to
		 * make sure it's always interpreted correctly.
		 */
		if (!(c & 0x80) && (isalnum(c) || c == '$' || c == '-' ||
		if (!(c & 0x80) && (isalnum(c) || strchr(allowed, c) != NULL))
		    c == '_' || c == '.' || c == '!' || c == '*' || c == '(' ||
		    c == ')' || c == ',' || strchr(ignored, c) != NULL))
			retCString[i++] = c;
		else {
			unsigned char high, low;

			high = c >> 4;
			low = c & 0x0F;

Modified src/OFURL.m from [ad4fe54b6f] to [6c7fd6999e].

83
84
85
86
87
88
89

90
91


92
93
94
95
96
97
98


99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122




123
124
125


126
127
128
129
130
131
132
133
134
135
136
137
138


139
140
141
142
143
144
145
146
147
148
149
150
151


152
153
154
155
156
157
158
159
160
161
162
163
164
165
166


167
168
169
170
171
172
173


174
175
176
177
178
179
180


181
182
183
184





185
186
187
188
189
190
191
83
84
85
86
87
88
89
90


91
92

93
94
95
96


97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118




119
120
121
122
123


124
125
126
127
128
129
130
131
132
133
134
135
136


137
138
139
140
141
142
143
144
145
146
147
148
149


150
151
152
153
154
155
156
157
158
159
160
161
162
163
164


165
166
167
168
169
170
171


172
173
174
175
176
177
178


179
180
181
182


183
184
185
186
187
188
189
190
191
192
193
194







+
-
-
+
+
-




-
-
+
+




















-
-
-
-
+
+
+
+

-
-
+
+











-
-
+
+











-
-
+
+













-
-
+
+





-
-
+
+





-
-
+
+


-
-
+
+
+
+
+








		if ((tmp = strstr(UTF8String, "://")) == NULL)
			@throw [OFInvalidFormatException exception];

		for (tmp2 = UTF8String; tmp2 < tmp; tmp2++)
			*tmp2 = tolower((unsigned char)*tmp2);

		_scheme = [[OFString alloc]
		_scheme = [[[OFString stringWithUTF8String: UTF8String
						    length: tmp - UTF8String]
		    initWithUTF8String: UTF8String
				length: tmp - UTF8String];
		    stringByURLDecoding] copy];

		UTF8String = tmp + 3;

		if ([_scheme isEqual: @"file"]) {
			_path = [[[OFString stringWithUTF8String:
			    UTF8String] stringByURLDecoding] copy];
			_path = [[OFString alloc]
			    initWithUTF8String: UTF8String];

			objc_autoreleasePoolPop(pool);
			return self;
		}

		if ((tmp = strchr(UTF8String, '/')) != NULL) {
			*tmp = '\0';
			tmp++;
		}

		if ((tmp2 = strchr(UTF8String, '@')) != NULL) {
			char *tmp3;

			*tmp2 = '\0';
			tmp2++;

			if ((tmp3 = strchr(UTF8String, ':')) != NULL) {
				*tmp3 = '\0';
				tmp3++;

				_user = [[[OFString stringWithUTF8String:
				    UTF8String] stringByURLDecoding] copy];
				_password = [[[OFString stringWithUTF8String:
				    tmp3] stringByURLDecoding] copy];
				_user = [[OFString alloc]
				    initWithUTF8String: UTF8String];
				_password = [[OFString alloc]
				    initWithUTF8String: tmp3];
			} else
				_user = [[[OFString stringWithUTF8String:
				    UTF8String] stringByURLDecoding] copy];
				_user = [[OFString alloc]
				    initWithUTF8String: UTF8String];

			UTF8String = tmp2;
		}

		if ((tmp2 = strchr(UTF8String, ':')) != NULL) {
			void *pool;
			OFString *portString;

			*tmp2 = '\0';
			tmp2++;

			_host = [[[OFString stringWithUTF8String:
			    UTF8String] stringByURLDecoding] copy];
			_host = [[OFString alloc]
			    initWithUTF8String: UTF8String];

			pool = objc_autoreleasePoolPush();
			portString = [OFString stringWithUTF8String: tmp2];

			if ([portString decimalValue] > 65535)
				@throw [OFInvalidFormatException exception];

			_port = [portString decimalValue];

			objc_autoreleasePoolPop(pool);
		} else {
			_host = [[[OFString stringWithUTF8String:
			    UTF8String] stringByURLDecoding] copy];
			_host = [[OFString alloc]
			    initWithUTF8String: UTF8String];

			if ([_scheme isEqual: @"http"])
				_port = 80;
			else if ([_scheme isEqual: @"https"])
				_port = 443;
			else if ([_scheme isEqual: @"ftp"])
				_port = 21;
		}

		if ((UTF8String = tmp) != NULL) {
			if ((tmp = strchr(UTF8String, '#')) != NULL) {
				*tmp = '\0';

				_fragment = [[[OFString stringWithUTF8String:
				    tmp + 1] stringByURLDecoding] copy];
				_fragment = [[OFString alloc]
				    initWithUTF8String: tmp + 1];
			}

			if ((tmp = strchr(UTF8String, '?')) != NULL) {
				*tmp = '\0';

				_query = [[[OFString stringWithUTF8String:
				    tmp + 1] stringByURLDecoding] copy];
				_query = [[OFString alloc]
				    initWithUTF8String: tmp + 1];
			}

			if ((tmp = strchr(UTF8String, ';')) != NULL) {
				*tmp = '\0';

				_parameters = [[[OFString stringWithUTF8String:
				    tmp + 1] stringByURLDecoding] copy];
				_parameters = [[OFString alloc]
				    initWithUTF8String: tmp + 1];
			}

			_path = [[[OFString stringWithUTF8String:
			    UTF8String] stringByURLDecoding] copy];
			UTF8String--;
			*UTF8String = '/';

			_path = [[OFString alloc]
			    initWithUTF8String: UTF8String];
		}

		objc_autoreleasePoolPop(pool);
	} @catch (id e) {
		[self release];
		@throw e;
	} @finally {
220
221
222
223
224
225
226
227
228


229
230
231
232
233
234


235
236
237
238
239
240


241
242
243
244
245


246
247
248
249

250
251
252
253
254
255
256
257
223
224
225
226
227
228
229


230
231
232
233
234
235


236
237
238
239
240
241


242
243
244
245
246


247
248
249
250
251

252

253
254
255
256
257
258
259







-
-
+
+




-
-
+
+




-
-
+
+



-
-
+
+



-
+
-







			     exceptionWithRequestedSize:
			     [string UTF8StringLength]];

		UTF8String = UTF8String2;

		if ((tmp = strchr(UTF8String, '#')) != NULL) {
			*tmp = '\0';
			_fragment = [[[OFString stringWithUTF8String:
			    tmp + 1] stringByURLDecoding] copy];
			_fragment = [[OFString alloc]
			    initWithUTF8String: tmp + 1];
		}

		if ((tmp = strchr(UTF8String, '?')) != NULL) {
			*tmp = '\0';
			_query = [[[OFString stringWithUTF8String:
			    tmp + 1] stringByURLDecoding] copy];
			_query = [[OFString alloc]
			    initWithUTF8String: tmp + 1];
		}

		if ((tmp = strchr(UTF8String, ';')) != NULL) {
			*tmp = '\0';
			_parameters = [[[OFString stringWithUTF8String:
			    tmp + 1] stringByURLDecoding] copy];
			_parameters = [[OFString alloc]
			    initWithUTF8String: tmp + 1];
		}

		if (*UTF8String == '/')
			_path = [[[OFString stringWithUTF8String:
			    UTF8String + 1] stringByURLDecoding] copy];
			_path = [[OFString alloc]
			    initWithUTF8String: UTF8String];
		else {
			OFString *path, *s;

			path = [[[OFString stringWithUTF8String:
			path = [OFString stringWithUTF8String: UTF8String];
			    UTF8String] stringByURLDecoding] copy];

			if ([URL->_path hasSuffix: @"/"])
				s = [URL->_path stringByAppendingString: path];
			else
				s = [OFString stringWithFormat: @"%@/../%@",
								URL->_path,
								path];
384
385
386
387
388
389
390
391

392
393
394
395

396
397
398
399
400
401
402
403

404
405
406
407

408
409
410

411
412
413
414
415
416
417
418
419
420







421
422

423
424
425

426
427
428

429
430
431
432
433
434
435
386
387
388
389
390
391
392

393
394
395
396

397

398
399
400
401
402
403

404


405

406
407
408

409
410
411
412
413
414
415




416
417
418
419
420
421
422
423

424
425
426

427
428
429

430
431
432
433
434
435
436
437







-
+



-
+
-






-
+
-
-

-
+


-
+






-
-
-
-
+
+
+
+
+
+
+

-
+


-
+


-
+







}

- (OFString*)string
{
	OFMutableString *ret = [OFMutableString string];
	void *pool = objc_autoreleasePoolPush();

	[ret appendFormat: @"%@://", [_scheme stringByURLEncoding]];
	[ret appendFormat: @"%@://", _scheme];

	if ([_scheme isEqual: @"file"]) {
		if (_path != nil)
			[ret appendString: [_path
			[ret appendString: _path];
			    stringByURLEncodingWithIgnoredCharacters: "/"]];

		objc_autoreleasePoolPop(pool);
		return ret;
	}

	if (_user != nil && _password != nil)
		[ret appendFormat: @"%@:%@@",
		[ret appendFormat: @"%@:%@@", _user, _password];
				   [_user stringByURLEncoding],
				   [_password stringByURLEncoding]];
	else if (_user != nil)
		[ret appendFormat: @"%@@", [_user stringByURLEncoding]];
		[ret appendFormat: @"%@@", _user];

	if (_host != nil)
		[ret appendString: [_host stringByURLEncoding]];
		[ret appendString: _host];

	if (!(([_scheme isEqual: @"http"] && _port == 80) ||
	    ([_scheme isEqual: @"https"] && _port == 443) ||
	    ([_scheme isEqual: @"ftp"] && _port == 21)))
		[ret appendFormat: @":%u", _port];

	if (_path != nil)
		[ret appendFormat: @"/%@",
		    [_path stringByURLEncodingWithIgnoredCharacters: "/"]];

	if (_path != nil) {
		if (![_path hasPrefix: @"/"])
			@throw [OFInvalidFormatException exception];

		[ret appendString: _path];
	}

	if (_parameters != nil)
		[ret appendFormat: @";%@", [_parameters stringByURLEncoding]];
		[ret appendFormat: @";%@", _parameters];

	if (_query != nil)
		[ret appendFormat: @"?%@", [_query stringByURLEncoding]];
		[ret appendFormat: @"?%@", _query];

	if (_fragment != nil)
		[ret appendFormat: @"#%@", [_fragment stringByURLEncoding]];
		[ret appendFormat: @"#%@", _fragment];

	objc_autoreleasePoolPop(pool);

	[ret makeImmutable];

	return ret;
}

Modified tests/OFURLTests.m from [b3ecd6e7b1] to [6d34fa57cb].

21
22
23
24
25
26
27
28

29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64



65
66
67


68
69
70

71
72
73
74



75
76

77
78
79
80
81
82
83
21
22
23
24
25
26
27

28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62


63
64
65
66


67
68
69
70

71
72
73


74
75
76
77

78
79
80
81
82
83
84
85







-
+

















-
+
















-
-
+
+
+

-
-
+
+


-
+


-
-
+
+
+

-
+







#import "OFAutoreleasePool.h"

#import "OFInvalidFormatException.h"

#import "TestsAppDelegate.h"

static OFString *module = @"OFURL";
static OFString *url_str = @"ht%3Atp://us%3Aer:p%40w@ho%3Ast:1234/"
static OFString *url_str = @"ht%3atp://us%3Aer:p%40w@ho%3Ast:1234/"
    @"pa%3Bth;pa%3Fram?que%23ry#frag%23ment";

@implementation TestsAppDelegate (OFURLTests)
- (void)URLTests
{
	OFAutoreleasePool *pool = [[OFAutoreleasePool alloc] init];
	OFURL *u1, *u2, *u3, *u4;

	TEST(@"+[URLWithString:]",
	    R(u1 = [OFURL URLWithString: url_str]) &&
	    R(u2 = [OFURL URLWithString: @"http://foo:80"]) &&
	    R(u3 = [OFURL URLWithString: @"http://bar/"]) &&
	    R(u4 = [OFURL URLWithString: @"file:///etc/passwd"]))

	TEST(@"+[URLWithString:relativeToURL:]",
	    [[[OFURL URLWithString: @"/foo"
		     relativeToURL: u1] string] isEqual:
	    @"ht%3Atp://us%3Aer:p%40w@ho%3Ast:1234/foo"] &&
	    @"ht%3atp://us%3Aer:p%40w@ho%3Ast:1234/foo"] &&
	    [[[OFURL URLWithString: @"foo/bar?q"
		     relativeToURL: [OFURL URLWithString: @"http://h/qux/quux"]]
	    string] isEqual: @"http://h/qux/foo/bar?q"] &&
	    [[[OFURL URLWithString: @"foo/bar"
		     relativeToURL: [OFURL URLWithString: @"http://h/qux/?x"]]
	    string] isEqual: @"http://h/qux/foo/bar"] &&
	    [[[OFURL URLWithString: @"http://foo/?q"
		     relativeToURL: u1] string] isEqual: @"http://foo/?q"])

	TEST(@"-[string]",
	    [[u1 string] isEqual: url_str] &&
	    [[u2 string] isEqual: @"http://foo"] &&
	    [[u3 string] isEqual: @"http://bar/"] &&
	    [[u4 string] isEqual: @"file:///etc/passwd"])

	TEST(@"-[scheme]",
	    [[u1 scheme] isEqual: @"ht:tp"] && [[u4 scheme] isEqual: @"file"])
	TEST(@"-[user]", [[u1 user] isEqual: @"us:er"] && [u4 user] == nil)
	    [[u1 scheme] isEqual: @"ht%3atp"] && [[u4 scheme] isEqual: @"file"])

	TEST(@"-[user]", [[u1 user] isEqual: @"us%3Aer"] && [u4 user] == nil)
	TEST(@"-[password]",
	    [[u1 password] isEqual: @"p@w"] && [u4 password] == nil)
	TEST(@"-[host]", [[u1 host] isEqual: @"ho:st"] && [u4 port] == 0)
	    [[u1 password] isEqual: @"p%40w"] && [u4 password] == nil)
	TEST(@"-[host]", [[u1 host] isEqual: @"ho%3Ast"] && [u4 port] == 0)
	TEST(@"-[port]", [u1 port] == 1234)
	TEST(@"-[path]",
	    [[u1 path] isEqual: @"pa;th"] &&
	    [[u1 path] isEqual: @"/pa%3Bth"] &&
	    [[u4 path] isEqual: @"/etc/passwd"])
	TEST(@"-[parameters]",
	    [[u1 parameters] isEqual: @"pa?ram"] && [u4 parameters] == nil)
	TEST(@"-[query]", [[u1 query] isEqual: @"que#ry"] && [u4 query] == nil)
	    [[u1 parameters] isEqual: @"pa%3Fram"] && [u4 parameters] == nil)
	TEST(@"-[query]",
	    [[u1 query] isEqual: @"que%23ry"] && [u4 query] == nil)
	TEST(@"-[fragment]",
	    [[u1 fragment] isEqual: @"frag#ment"] && [u4 fragment] == nil)
	    [[u1 fragment] isEqual: @"frag%23ment"] && [u4 fragment] == nil)

	TEST(@"-[copy]", R(u4 = [[u1 copy] autorelease]))

	TEST(@"-[isEqual:]", [u1 isEqual: u4] && ![u2 isEqual: u3] &&
	    [[OFURL URLWithString: @"HTTP://bar/"] isEqual: u3])

	TEST(@"-[hash:]", [u1 hash] == [u4 hash] && [u2 hash] != [u3 hash])

Modified tests/serialization.xml from [d1cce18711] to [2be1e47e3a].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
1
2
3
4
5
6
7
8
9
10



















11
12
13
14
15
16
17










-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-







<?xml version='1.0' encoding='UTF-8'?>
<serialization xmlns='https://webkeks.org/objfw/serialization' version='1'>
  <OFMutableDictionary>
    <key>
      <OFString>Blub</OFString>
    </key>
    <object>
      <OFString>B&quot;la</OFString>
    </object>
    <key>
      <OFDataArray>MDEyMzQ1Njc4OTo7PEFCQ0RFRkdISklLTE1OT1BRUlNUVVZXWFla</OFDataArray>
    </key>
    <object>
      <OFString>data</OFString>
    </object>
    <key>
      <OFArray>
        <OFString>Qu&quot;xbar
test</OFString>
        <OFNumber type='signed'>1234</OFNumber>
        <OFNumber type='double'>40934a456d5cfaad</OFNumber>
        <OFMutableString>asd</OFMutableString>
        <OFDate>40934a456d5cfaad</OFDate>
      </OFArray>
    </key>
    <object>
      <OFString>Hello</OFString>
    </object>
    <key>
      <OFList>
        <OFString>Hello</OFString>
        <OFString>Wo&#xD;ld!
How are you?</OFString>
        <OFURL>https://webkeks.org/</OFURL>
        <OFXMLElement name='x'>
          <children>
53
54
55
56
57
58
59



















60
61
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+


          </object>
        </OFCountedSet>
      </OFList>
    </key>
    <object>
      <OFString>list</OFString>
    </object>
    <key>
      <OFDataArray>MDEyMzQ1Njc4OTo7PEFCQ0RFRkdISklLTE1OT1BRUlNUVVZXWFla</OFDataArray>
    </key>
    <object>
      <OFString>data</OFString>
    </object>
    <key>
      <OFArray>
        <OFString>Qu&quot;xbar
test</OFString>
        <OFNumber type='signed'>1234</OFNumber>
        <OFNumber type='double'>40934a456d5cfaad</OFNumber>
        <OFMutableString>asd</OFMutableString>
        <OFDate>40934a456d5cfaad</OFDate>
      </OFArray>
    </key>
    <object>
      <OFString>Hello</OFString>
    </object>
  </OFMutableDictionary>
</serialization>