1 |
From f9e0f439b72e0b2fb035be1bc60aaceeed7f6ed0 Mon Sep 17 00:00:00 2001 |
2 |
From: Leonhard Holz <leonhard.holz@web.de> |
3 |
Date: Tue, 13 Jan 2015 11:33:56 +0530 |
4 |
Subject: [PATCH 16/18] Fix memory handling in strxfrm_l [BZ #16009] |
5 |
|
6 |
[Modified from the original email by Siddhesh Poyarekar] |
7 |
|
8 |
This patch solves bug #16009 by implementing an additional path in |
9 |
strxfrm that does not depend on caching the weight and rule indices. |
10 |
|
11 |
In detail the following changed: |
12 |
|
13 |
* The old main loop was factored out of strxfrm_l into the function |
14 |
do_xfrm_cached to be able to alternativly use the non-caching version |
15 |
do_xfrm. |
16 |
|
17 |
* strxfrm_l allocates a a fixed size array on the stack. If this is not |
18 |
sufficiant to store the weight and rule indices, the non-caching path is |
19 |
taken. As the cache size is not dependent on the input there can be no |
20 |
problems with integer overflows or stack allocations greater than |
21 |
__MAX_ALLOCA_CUTOFF. Note that malloc-ing is not possible because the |
22 |
definition of strxfrm does not allow an oom errorhandling. |
23 |
|
24 |
* The uncached path determines the weight and rule index for every char |
25 |
and for every pass again. |
26 |
|
27 |
* Passing all the locale data array by array resulted in very long |
28 |
parameter lists, so I introduced a structure that holds them. |
29 |
|
30 |
* Checking for zero src string has been moved a bit upwards, it is |
31 |
before the locale data initialization now. |
32 |
|
33 |
* To verify that the non-caching path works correct I added a test run |
34 |
to localedata/sort-test.sh & localedata/xfrm-test.c where all strings |
35 |
are patched up with spaces so that they are too large for the caching path. |
36 |
|
37 |
(cherry picked from commit 0f9e585480edcdf1e30dc3d79e24b84aeee516fa) |
38 |
|
39 |
Conflicts: |
40 |
ChangeLog |
41 |
NEWS |
42 |
--- |
43 |
ChangeLog | 16 ++ |
44 |
NEWS | 2 +- |
45 |
localedata/sort-test.sh | 7 + |
46 |
localedata/xfrm-test.c | 52 +++++- |
47 |
string/strxfrm_l.c | 488 ++++++++++++++++++++++++++++++++++++++---------- |
48 |
5 files changed, 464 insertions(+), 101 deletions(-) |
49 |
|
50 |
diff --git a/ChangeLog b/ChangeLog |
51 |
index dbf7e86..7a2e6c9 100644 |
52 |
--- a/ChangeLog |
53 |
+++ b/ChangeLog |
54 |
@@ -1,3 +1,19 @@ |
55 |
+2015-02-16 Leonhard Holz <leonhard.holz@web.de> |
56 |
+ |
57 |
+ [BZ #16009] |
58 |
+ * string/strxfrm_l.c (STRXFRM): Allocate fixed size cache for |
59 |
+ weights and rules. Use do_xfrm_cached if data fits in cache, |
60 |
+ do_xfrm otherwise. Moved former main loop to... |
61 |
+ * (do_xfrm_cached): New function. |
62 |
+ * (do_xfrm): Non-caching version of do_xfrm_cached. Uses |
63 |
+ find_idx, find_position and stack_push. |
64 |
+ * (find_idx): New function. |
65 |
+ * (find_position): Likewise. |
66 |
+ * localedata/sort-test.sh: Added test run for do_xfrm. |
67 |
+ * localedata/xfrm-test.c (main): Added command line option |
68 |
+ -nocache to run the test with strings that are too large for |
69 |
+ the STRXFRM cache. |
70 |
+ |
71 |
2015-02-16 Kostya Serebryany <konstantin.s.serebryany@gmail.com> |
72 |
Roland McGrath <roland@hack.frob.com> |
73 |
|
74 |
diff --git a/NEWS b/NEWS |
75 |
index 9bc835c..f578805 100644 |
76 |
--- a/NEWS |
77 |
+++ b/NEWS |
78 |
@@ -9,7 +9,7 @@ Version 2.20.1 |
79 |
|
80 |
* The following bugs are resolved with this release: |
81 |
|
82 |
- 16617, 17266, 17370, 17371, 17460, 17485, 17555, 17625, 17630. |
83 |
+ 16009, 16617, 17266, 17370, 17371, 17460, 17485, 17555, 17625, 17630. |
84 |
|
85 |
* CVE-2014-7817 The wordexp function could ignore the WRDE_NOCMD flag |
86 |
under certain input conditions resulting in the execution of a shell for |
87 |
diff --git a/localedata/sort-test.sh b/localedata/sort-test.sh |
88 |
index e37129a..3cb57fb 100644 |
89 |
--- a/localedata/sort-test.sh |
90 |
+++ b/localedata/sort-test.sh |
91 |
@@ -53,11 +53,18 @@ for l in $lang; do |
92 |
${common_objpfx}localedata/xfrm-test $id < $cns.in \ |
93 |
> ${common_objpfx}localedata/$cns.xout || here=1 |
94 |
cmp -s $cns.in ${common_objpfx}localedata/$cns.xout || here=1 |
95 |
+ ${test_program_prefix_before_env} \ |
96 |
+ ${run_program_env} \ |
97 |
+ LC_ALL=$l ${test_program_prefix_after_env} \ |
98 |
+ ${common_objpfx}localedata/xfrm-test $id -nocache < $cns.in \ |
99 |
+ > ${common_objpfx}localedata/$cns.nocache.xout || here=1 |
100 |
+ cmp -s $cns.in ${common_objpfx}localedata/$cns.nocache.xout || here=1 |
101 |
if test $here -eq 0; then |
102 |
echo "$l xfrm-test OK" |
103 |
else |
104 |
echo "$l xfrm-test FAIL" |
105 |
diff -u $cns.in ${common_objpfx}localedata/$cns.xout | sed 's/^/ /' |
106 |
+ diff -u $cns.in ${common_objpfx}localedata/$cns.nocache.xout | sed 's/^/ /' |
107 |
status=1 |
108 |
fi |
109 |
done |
110 |
diff --git a/localedata/xfrm-test.c b/localedata/xfrm-test.c |
111 |
index d2aba7d..5cf29f6 100644 |
112 |
--- a/localedata/xfrm-test.c |
113 |
+++ b/localedata/xfrm-test.c |
114 |
@@ -23,7 +23,10 @@ |
115 |
#include <stdio.h> |
116 |
#include <stdlib.h> |
117 |
#include <string.h> |
118 |
+#include <stdbool.h> |
119 |
|
120 |
+/* Keep in sync with string/strxfrm_l.c. */ |
121 |
+#define SMALL_STR_SIZE 4095 |
122 |
|
123 |
struct lines |
124 |
{ |
125 |
@@ -37,6 +40,7 @@ int |
126 |
main (int argc, char *argv[]) |
127 |
{ |
128 |
int result = 0; |
129 |
+ bool nocache = false; |
130 |
size_t nstrings, nstrings_max; |
131 |
struct lines *strings; |
132 |
char *line = NULL; |
133 |
@@ -44,7 +48,18 @@ main (int argc, char *argv[]) |
134 |
size_t n; |
135 |
|
136 |
if (argc < 2) |
137 |
- error (1, 0, "usage: %s <random seed>", argv[0]); |
138 |
+ error (1, 0, "usage: %s <random seed> [-nocache]", argv[0]); |
139 |
+ |
140 |
+ if (argc == 3) |
141 |
+ { |
142 |
+ if (strcmp (argv[2], "-nocache") == 0) |
143 |
+ nocache = true; |
144 |
+ else |
145 |
+ { |
146 |
+ printf ("Unknown option %s!\n", argv[2]); |
147 |
+ exit (1); |
148 |
+ } |
149 |
+ } |
150 |
|
151 |
setlocale (LC_ALL, ""); |
152 |
|
153 |
@@ -59,9 +74,9 @@ main (int argc, char *argv[]) |
154 |
|
155 |
while (1) |
156 |
{ |
157 |
- char saved, *newp; |
158 |
- int needed; |
159 |
- int l; |
160 |
+ char saved, *word, *newp; |
161 |
+ size_t l, line_len, needed; |
162 |
+ |
163 |
if (getline (&line, &len, stdin) < 0) |
164 |
break; |
165 |
|
166 |
@@ -83,10 +98,35 @@ main (int argc, char *argv[]) |
167 |
|
168 |
saved = line[l]; |
169 |
line[l] = '\0'; |
170 |
- needed = strxfrm (NULL, line, 0); |
171 |
+ |
172 |
+ if (nocache) |
173 |
+ { |
174 |
+ line_len = strlen (line); |
175 |
+ word = malloc (line_len + SMALL_STR_SIZE + 1); |
176 |
+ if (word == NULL) |
177 |
+ { |
178 |
+ printf ("malloc failed: %m\n"); |
179 |
+ exit (1); |
180 |
+ } |
181 |
+ memset (word, ' ', SMALL_STR_SIZE); |
182 |
+ memcpy (word + SMALL_STR_SIZE, line, line_len); |
183 |
+ word[line_len + SMALL_STR_SIZE] = '\0'; |
184 |
+ } |
185 |
+ else |
186 |
+ word = line; |
187 |
+ |
188 |
+ needed = strxfrm (NULL, word, 0); |
189 |
newp = malloc (needed + 1); |
190 |
- strxfrm (newp, line, needed + 1); |
191 |
+ if (newp == NULL) |
192 |
+ { |
193 |
+ printf ("malloc failed: %m\n"); |
194 |
+ exit (1); |
195 |
+ } |
196 |
+ strxfrm (newp, word, needed + 1); |
197 |
strings[nstrings].xfrm = newp; |
198 |
+ |
199 |
+ if (nocache) |
200 |
+ free (word); |
201 |
line[l] = saved; |
202 |
++nstrings; |
203 |
} |
204 |
diff --git a/string/strxfrm_l.c b/string/strxfrm_l.c |
205 |
index 2d3f1bd..95ffd6f 100644 |
206 |
--- a/string/strxfrm_l.c |
207 |
+++ b/string/strxfrm_l.c |
208 |
@@ -40,9 +40,24 @@ |
209 |
#define CONCAT(a,b) CONCAT1(a,b) |
210 |
#define CONCAT1(a,b) a##b |
211 |
|
212 |
+/* Maximum string size that is calculated with cached indices. Right now this |
213 |
+ is an arbitrary value open to optimizations. SMALL_STR_SIZE * 4 has to be |
214 |
+ lower than __MAX_ALLOCA_CUTOFF. Keep localedata/xfrm-test.c in sync. */ |
215 |
+#define SMALL_STR_SIZE 4095 |
216 |
+ |
217 |
#include "../locale/localeinfo.h" |
218 |
#include WEIGHT_H |
219 |
|
220 |
+/* Group locale data for shorter parameter lists. */ |
221 |
+typedef struct |
222 |
+{ |
223 |
+ uint_fast32_t nrules; |
224 |
+ unsigned char *rulesets; |
225 |
+ USTRING_TYPE *weights; |
226 |
+ int32_t *table; |
227 |
+ USTRING_TYPE *extra; |
228 |
+ int32_t *indirect; |
229 |
+} locale_data_t; |
230 |
|
231 |
#ifndef WIDE_CHAR_VERSION |
232 |
|
233 |
@@ -81,113 +96,325 @@ utf8_encode (char *buf, int val) |
234 |
} |
235 |
#endif |
236 |
|
237 |
+/* Find next weight and rule index. Inlined since called for every char. */ |
238 |
+static __always_inline size_t |
239 |
+find_idx (const USTRING_TYPE **us, int32_t *weight_idx, |
240 |
+ unsigned char *rule_idx, const locale_data_t *l_data, const int pass) |
241 |
+{ |
242 |
+ int32_t tmp = findidx (l_data->table, l_data->indirect, l_data->extra, us, |
243 |
+ -1); |
244 |
+ *rule_idx = tmp >> 24; |
245 |
+ int32_t idx = tmp & 0xffffff; |
246 |
+ size_t len = l_data->weights[idx++]; |
247 |
+ |
248 |
+ /* Skip over indices of previous levels. */ |
249 |
+ for (int i = 0; i < pass; i++) |
250 |
+ { |
251 |
+ idx += len; |
252 |
+ len = l_data->weights[idx++]; |
253 |
+ } |
254 |
|
255 |
-size_t |
256 |
-STRXFRM (STRING_TYPE *dest, const STRING_TYPE *src, size_t n, __locale_t l) |
257 |
+ *weight_idx = idx; |
258 |
+ return len; |
259 |
+} |
260 |
+ |
261 |
+static int |
262 |
+find_position (const USTRING_TYPE *us, const locale_data_t *l_data, |
263 |
+ const int pass) |
264 |
{ |
265 |
- struct __locale_data *current = l->__locales[LC_COLLATE]; |
266 |
- uint_fast32_t nrules = current->values[_NL_ITEM_INDEX (_NL_COLLATE_NRULES)].word; |
267 |
- /* We don't assign the following values right away since it might be |
268 |
- unnecessary in case there are no rules. */ |
269 |
- const unsigned char *rulesets; |
270 |
- const int32_t *table; |
271 |
- const USTRING_TYPE *weights; |
272 |
- const USTRING_TYPE *extra; |
273 |
- const int32_t *indirect; |
274 |
+ int32_t weight_idx; |
275 |
+ unsigned char rule_idx; |
276 |
+ const USTRING_TYPE *usrc = us; |
277 |
+ |
278 |
+ find_idx (&usrc, &weight_idx, &rule_idx, l_data, pass); |
279 |
+ return l_data->rulesets[rule_idx * l_data->nrules + pass] & sort_position; |
280 |
+} |
281 |
+ |
282 |
+/* Do the transformation. */ |
283 |
+static size_t |
284 |
+do_xfrm (const USTRING_TYPE *usrc, STRING_TYPE *dest, size_t n, |
285 |
+ const locale_data_t *l_data) |
286 |
+{ |
287 |
+ int32_t weight_idx; |
288 |
+ unsigned char rule_idx; |
289 |
uint_fast32_t pass; |
290 |
- size_t needed; |
291 |
+ size_t needed = 0; |
292 |
size_t last_needed; |
293 |
- const USTRING_TYPE *usrc; |
294 |
- size_t srclen = STRLEN (src); |
295 |
- int32_t *idxarr; |
296 |
- unsigned char *rulearr; |
297 |
- size_t idxmax; |
298 |
- size_t idxcnt; |
299 |
- int use_malloc; |
300 |
|
301 |
- if (nrules == 0) |
302 |
+ /* Now the passes over the weights. */ |
303 |
+ for (pass = 0; pass < l_data->nrules; ++pass) |
304 |
{ |
305 |
- if (n != 0) |
306 |
- STPNCPY (dest, src, MIN (srclen + 1, n)); |
307 |
+ size_t backw_len = 0; |
308 |
+ last_needed = needed; |
309 |
+ const USTRING_TYPE *cur = usrc; |
310 |
+ const USTRING_TYPE *backw_start = NULL; |
311 |
|
312 |
- return srclen; |
313 |
- } |
314 |
+ /* We assume that if a rule has defined `position' in one section |
315 |
+ this is true for all of them. */ |
316 |
+ int position = find_position (cur, l_data, pass); |
317 |
|
318 |
- rulesets = (const unsigned char *) |
319 |
- current->values[_NL_ITEM_INDEX (_NL_COLLATE_RULESETS)].string; |
320 |
- table = (const int32_t *) |
321 |
- current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_TABLE,SUFFIX))].string; |
322 |
- weights = (const USTRING_TYPE *) |
323 |
- current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_WEIGHT,SUFFIX))].string; |
324 |
- extra = (const USTRING_TYPE *) |
325 |
- current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_EXTRA,SUFFIX))].string; |
326 |
- indirect = (const int32_t *) |
327 |
- current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_INDIRECT,SUFFIX))].string; |
328 |
- use_malloc = 0; |
329 |
+ if (position == 0) |
330 |
+ { |
331 |
+ while (*cur != L('\0')) |
332 |
+ { |
333 |
+ const USTRING_TYPE *pos = cur; |
334 |
+ size_t len = find_idx (&cur, &weight_idx, &rule_idx, l_data, |
335 |
+ pass); |
336 |
+ int rule = l_data->rulesets[rule_idx * l_data->nrules + pass]; |
337 |
|
338 |
- assert (((uintptr_t) table) % __alignof__ (table[0]) == 0); |
339 |
- assert (((uintptr_t) weights) % __alignof__ (weights[0]) == 0); |
340 |
- assert (((uintptr_t) extra) % __alignof__ (extra[0]) == 0); |
341 |
- assert (((uintptr_t) indirect) % __alignof__ (indirect[0]) == 0); |
342 |
+ if ((rule & sort_forward) != 0) |
343 |
+ { |
344 |
+ /* Handle the pushed backward sequence. */ |
345 |
+ if (backw_start != NULL) |
346 |
+ { |
347 |
+ for (size_t i = backw_len; i > 0; ) |
348 |
+ { |
349 |
+ int32_t weight_idx; |
350 |
+ unsigned char rule_idx; |
351 |
+ size_t len = find_idx (&backw_start, &weight_idx, |
352 |
+ &rule_idx, l_data, pass); |
353 |
+ if (needed + i < n) |
354 |
+ for (size_t j = len; j > 0; j--) |
355 |
+ dest[needed + i - j] = |
356 |
+ l_data->weights[weight_idx++]; |
357 |
+ |
358 |
+ i -= len; |
359 |
+ } |
360 |
|
361 |
- /* Handle an empty string as a special case. */ |
362 |
- if (srclen == 0) |
363 |
- { |
364 |
- if (n != 0) |
365 |
- *dest = L('\0'); |
366 |
- return 0; |
367 |
- } |
368 |
+ needed += backw_len; |
369 |
+ backw_start = NULL; |
370 |
+ backw_len = 0; |
371 |
+ } |
372 |
|
373 |
- /* We need the elements of the string as unsigned values since they |
374 |
- are used as indeces. */ |
375 |
- usrc = (const USTRING_TYPE *) src; |
376 |
- |
377 |
- /* Perform the first pass over the string and while doing this find |
378 |
- and store the weights for each character. Since we want this to |
379 |
- be as fast as possible we are using `alloca' to store the temporary |
380 |
- values. But since there is no limit on the length of the string |
381 |
- we have to use `malloc' if the string is too long. We should be |
382 |
- very conservative here. */ |
383 |
- if (! __libc_use_alloca ((srclen + 1) * (sizeof (int32_t) + 1))) |
384 |
- { |
385 |
- idxarr = (int32_t *) malloc ((srclen + 1) * (sizeof (int32_t) + 1)); |
386 |
- rulearr = (unsigned char *) &idxarr[srclen]; |
387 |
- |
388 |
- if (idxarr == NULL) |
389 |
- /* No memory. Well, go with the stack then. |
390 |
- |
391 |
- XXX Once this implementation is stable we will handle this |
392 |
- differently. Instead of precomputing the indeces we will |
393 |
- do this in time. This means, though, that this happens for |
394 |
- every pass again. */ |
395 |
- goto try_stack; |
396 |
- use_malloc = 1; |
397 |
- } |
398 |
- else |
399 |
- { |
400 |
- try_stack: |
401 |
- idxarr = (int32_t *) alloca (srclen * sizeof (int32_t)); |
402 |
- rulearr = (unsigned char *) alloca (srclen + 1); |
403 |
+ /* Now handle the forward element. */ |
404 |
+ if (needed + len < n) |
405 |
+ while (len-- > 0) |
406 |
+ dest[needed++] = l_data->weights[weight_idx++]; |
407 |
+ else |
408 |
+ /* No more characters fit into the buffer. */ |
409 |
+ needed += len; |
410 |
+ } |
411 |
+ else |
412 |
+ { |
413 |
+ /* Remember start of the backward sequence & track length. */ |
414 |
+ if (backw_start == NULL) |
415 |
+ backw_start = pos; |
416 |
+ backw_len += len; |
417 |
+ } |
418 |
+ } |
419 |
+ |
420 |
+ |
421 |
+ /* Handle the pushed backward sequence. */ |
422 |
+ if (backw_start != NULL) |
423 |
+ { |
424 |
+ for (size_t i = backw_len; i > 0; ) |
425 |
+ { |
426 |
+ size_t len = find_idx (&backw_start, &weight_idx, &rule_idx, |
427 |
+ l_data, pass); |
428 |
+ if (needed + i < n) |
429 |
+ for (size_t j = len; j > 0; j--) |
430 |
+ dest[needed + i - j] = |
431 |
+ l_data->weights[weight_idx++]; |
432 |
+ |
433 |
+ i -= len; |
434 |
+ } |
435 |
+ |
436 |
+ needed += backw_len; |
437 |
+ } |
438 |
+ } |
439 |
+ else |
440 |
+ { |
441 |
+ int val = 1; |
442 |
+#ifndef WIDE_CHAR_VERSION |
443 |
+ char buf[7]; |
444 |
+ size_t buflen; |
445 |
+#endif |
446 |
+ size_t i; |
447 |
+ |
448 |
+ while (*cur != L('\0')) |
449 |
+ { |
450 |
+ const USTRING_TYPE *pos = cur; |
451 |
+ size_t len = find_idx (&cur, &weight_idx, &rule_idx, l_data, |
452 |
+ pass); |
453 |
+ int rule = l_data->rulesets[rule_idx * l_data->nrules + pass]; |
454 |
+ |
455 |
+ if ((rule & sort_forward) != 0) |
456 |
+ { |
457 |
+ /* Handle the pushed backward sequence. */ |
458 |
+ if (backw_start != NULL) |
459 |
+ { |
460 |
+ for (size_t p = backw_len; p > 0; p--) |
461 |
+ { |
462 |
+ size_t len; |
463 |
+ int32_t weight_idx; |
464 |
+ unsigned char rule_idx; |
465 |
+ const USTRING_TYPE *backw_cur = backw_start; |
466 |
+ |
467 |
+ /* To prevent a warning init the used vars. */ |
468 |
+ len = find_idx (&backw_cur, &weight_idx, |
469 |
+ &rule_idx, l_data, pass); |
470 |
+ |
471 |
+ for (i = 1; i < p; i++) |
472 |
+ len = find_idx (&backw_cur, &weight_idx, |
473 |
+ &rule_idx, l_data, pass); |
474 |
+ |
475 |
+ if (len != 0) |
476 |
+ { |
477 |
+#ifdef WIDE_CHAR_VERSION |
478 |
+ if (needed + 1 + len < n) |
479 |
+ { |
480 |
+ dest[needed] = val; |
481 |
+ for (i = 0; i < len; ++i) |
482 |
+ dest[needed + 1 + i] = |
483 |
+ l_data->weights[weight_idx + i]; |
484 |
+ } |
485 |
+ needed += 1 + len; |
486 |
+#else |
487 |
+ buflen = utf8_encode (buf, val); |
488 |
+ if (needed + buflen + len < n) |
489 |
+ { |
490 |
+ for (i = 0; i < buflen; ++i) |
491 |
+ dest[needed + i] = buf[i]; |
492 |
+ for (i = 0; i < len; ++i) |
493 |
+ dest[needed + buflen + i] = |
494 |
+ l_data->weights[weight_idx + i]; |
495 |
+ } |
496 |
+ needed += buflen + len; |
497 |
+#endif |
498 |
+ val = 1; |
499 |
+ } |
500 |
+ else |
501 |
+ ++val; |
502 |
+ } |
503 |
+ |
504 |
+ backw_start = NULL; |
505 |
+ backw_len = 0; |
506 |
+ } |
507 |
+ |
508 |
+ /* Now handle the forward element. */ |
509 |
+ if (len != 0) |
510 |
+ { |
511 |
+#ifdef WIDE_CHAR_VERSION |
512 |
+ if (needed + 1 + len < n) |
513 |
+ { |
514 |
+ dest[needed] = val; |
515 |
+ for (i = 0; i < len; ++i) |
516 |
+ dest[needed + 1 + i] = |
517 |
+ l_data->weights[weight_idx + i]; |
518 |
+ } |
519 |
+ needed += 1 + len; |
520 |
+#else |
521 |
+ buflen = utf8_encode (buf, val); |
522 |
+ if (needed + buflen + len < n) |
523 |
+ { |
524 |
+ for (i = 0; i < buflen; ++i) |
525 |
+ dest[needed + i] = buf[i]; |
526 |
+ for (i = 0; i < len; ++i) |
527 |
+ dest[needed + buflen + i] = |
528 |
+ l_data->weights[weight_idx + i]; |
529 |
+ } |
530 |
+ needed += buflen + len; |
531 |
+#endif |
532 |
+ val = 1; |
533 |
+ } |
534 |
+ else |
535 |
+ ++val; |
536 |
+ } |
537 |
+ else |
538 |
+ { |
539 |
+ /* Remember start of the backward sequence & track length. */ |
540 |
+ if (backw_start == NULL) |
541 |
+ backw_start = pos; |
542 |
+ backw_len++; |
543 |
+ } |
544 |
+ } |
545 |
+ |
546 |
+ /* Handle the pushed backward sequence. */ |
547 |
+ if (backw_start != NULL) |
548 |
+ { |
549 |
+ for (size_t p = backw_len; p > 0; p--) |
550 |
+ { |
551 |
+ size_t len; |
552 |
+ int32_t weight_idx; |
553 |
+ unsigned char rule_idx; |
554 |
+ const USTRING_TYPE *backw_cur = backw_start; |
555 |
+ |
556 |
+ /* To prevent a warning init the used vars. */ |
557 |
+ len = find_idx (&backw_cur, &weight_idx, |
558 |
+ &rule_idx, l_data, pass); |
559 |
+ |
560 |
+ for (i = 1; i < p; i++) |
561 |
+ len = find_idx (&backw_cur, &weight_idx, |
562 |
+ &rule_idx, l_data, pass); |
563 |
+ |
564 |
+ if (len != 0) |
565 |
+ { |
566 |
+#ifdef WIDE_CHAR_VERSION |
567 |
+ if (needed + 1 + len < n) |
568 |
+ { |
569 |
+ dest[needed] = val; |
570 |
+ for (i = 0; i < len; ++i) |
571 |
+ dest[needed + 1 + i] = |
572 |
+ l_data->weights[weight_idx + i]; |
573 |
+ } |
574 |
+ needed += 1 + len; |
575 |
+#else |
576 |
+ buflen = utf8_encode (buf, val); |
577 |
+ if (needed + buflen + len < n) |
578 |
+ { |
579 |
+ for (i = 0; i < buflen; ++i) |
580 |
+ dest[needed + i] = buf[i]; |
581 |
+ for (i = 0; i < len; ++i) |
582 |
+ dest[needed + buflen + i] = |
583 |
+ l_data->weights[weight_idx + i]; |
584 |
+ } |
585 |
+ needed += buflen + len; |
586 |
+#endif |
587 |
+ val = 1; |
588 |
+ } |
589 |
+ else |
590 |
+ ++val; |
591 |
+ } |
592 |
+ } |
593 |
+ } |
594 |
+ |
595 |
+ /* Finally store the byte to separate the passes or terminate |
596 |
+ the string. */ |
597 |
+ if (needed < n) |
598 |
+ dest[needed] = pass + 1 < l_data->nrules ? L('\1') : L('\0'); |
599 |
+ ++needed; |
600 |
} |
601 |
|
602 |
- idxmax = 0; |
603 |
- do |
604 |
+ /* This is a little optimization: many collation specifications have |
605 |
+ a `position' rule at the end and if no non-ignored character |
606 |
+ is found the last \1 byte is immediately followed by a \0 byte |
607 |
+ signalling this. We can avoid the \1 byte(s). */ |
608 |
+ if (needed > 2 && needed == last_needed + 1) |
609 |
{ |
610 |
- int32_t tmp = findidx (table, indirect, extra, &usrc, -1); |
611 |
- rulearr[idxmax] = tmp >> 24; |
612 |
- idxarr[idxmax] = tmp & 0xffffff; |
613 |
- |
614 |
- ++idxmax; |
615 |
+ /* Remove the \1 byte. */ |
616 |
+ if (--needed <= n) |
617 |
+ dest[needed - 1] = L('\0'); |
618 |
} |
619 |
- while (*usrc != L('\0')); |
620 |
|
621 |
- /* This element is only read, the value never used but to determine |
622 |
- another value which then is ignored. */ |
623 |
- rulearr[idxmax] = '\0'; |
624 |
+ /* Return the number of bytes/words we need, but don't count the NUL |
625 |
+ byte/word at the end. */ |
626 |
+ return needed - 1; |
627 |
+} |
628 |
+ |
629 |
+/* Do the transformation using weight-index and rule cache. */ |
630 |
+static size_t |
631 |
+do_xfrm_cached (STRING_TYPE *dest, size_t n, const locale_data_t *l_data, |
632 |
+ size_t idxmax, int32_t *idxarr, const unsigned char *rulearr) |
633 |
+{ |
634 |
+ uint_fast32_t nrules = l_data->nrules; |
635 |
+ unsigned char *rulesets = l_data->rulesets; |
636 |
+ USTRING_TYPE *weights = l_data->weights; |
637 |
+ uint_fast32_t pass; |
638 |
+ size_t needed = 0; |
639 |
+ size_t last_needed; |
640 |
+ size_t idxcnt; |
641 |
|
642 |
- /* Now the passes over the weights. We now use the indeces we found |
643 |
- before. */ |
644 |
- needed = 0; |
645 |
+ /* Now the passes over the weights. */ |
646 |
for (pass = 0; pass < nrules; ++pass) |
647 |
{ |
648 |
size_t backw_stop = ~0ul; |
649 |
@@ -433,14 +660,87 @@ STRXFRM (STRING_TYPE *dest, const STRING_TYPE *src, size_t n, __locale_t l) |
650 |
dest[needed - 1] = L('\0'); |
651 |
} |
652 |
|
653 |
- /* Free the memory if needed. */ |
654 |
- if (use_malloc) |
655 |
- free (idxarr); |
656 |
- |
657 |
/* Return the number of bytes/words we need, but don't count the NUL |
658 |
byte/word at the end. */ |
659 |
return needed - 1; |
660 |
} |
661 |
+ |
662 |
+size_t |
663 |
+STRXFRM (STRING_TYPE *dest, const STRING_TYPE *src, size_t n, __locale_t l) |
664 |
+{ |
665 |
+ locale_data_t l_data; |
666 |
+ struct __locale_data *current = l->__locales[LC_COLLATE]; |
667 |
+ l_data.nrules = current->values[_NL_ITEM_INDEX (_NL_COLLATE_NRULES)].word; |
668 |
+ |
669 |
+ /* Handle byte comparison case. */ |
670 |
+ if (l_data.nrules == 0) |
671 |
+ { |
672 |
+ size_t srclen = STRLEN (src); |
673 |
+ |
674 |
+ if (n != 0) |
675 |
+ STPNCPY (dest, src, MIN (srclen + 1, n)); |
676 |
+ |
677 |
+ return srclen; |
678 |
+ } |
679 |
+ |
680 |
+ /* Handle an empty string, code hereafter relies on strlen (src) > 0. */ |
681 |
+ if (*src == L('\0')) |
682 |
+ { |
683 |
+ if (n != 0) |
684 |
+ *dest = L('\0'); |
685 |
+ return 0; |
686 |
+ } |
687 |
+ |
688 |
+ /* Get the locale data. */ |
689 |
+ l_data.rulesets = (unsigned char *) |
690 |
+ current->values[_NL_ITEM_INDEX (_NL_COLLATE_RULESETS)].string; |
691 |
+ l_data.table = (int32_t *) |
692 |
+ current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_TABLE,SUFFIX))].string; |
693 |
+ l_data.weights = (USTRING_TYPE *) |
694 |
+ current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_WEIGHT,SUFFIX))].string; |
695 |
+ l_data.extra = (USTRING_TYPE *) |
696 |
+ current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_EXTRA,SUFFIX))].string; |
697 |
+ l_data.indirect = (int32_t *) |
698 |
+ current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_INDIRECT,SUFFIX))].string; |
699 |
+ |
700 |
+ assert (((uintptr_t) l_data.table) % __alignof__ (l_data.table[0]) == 0); |
701 |
+ assert (((uintptr_t) l_data.weights) % __alignof__ (l_data.weights[0]) == 0); |
702 |
+ assert (((uintptr_t) l_data.extra) % __alignof__ (l_data.extra[0]) == 0); |
703 |
+ assert (((uintptr_t) l_data.indirect) % __alignof__ (l_data.indirect[0]) == 0); |
704 |
+ |
705 |
+ /* We need the elements of the string as unsigned values since they |
706 |
+ are used as indeces. */ |
707 |
+ const USTRING_TYPE *usrc = (const USTRING_TYPE *) src; |
708 |
+ |
709 |
+ /* Allocate cache for small strings on the stack and fill it with weight and |
710 |
+ rule indices. If the cache size is not sufficient, continue with the |
711 |
+ uncached xfrm version. */ |
712 |
+ size_t idxmax = 0; |
713 |
+ const USTRING_TYPE *cur = usrc; |
714 |
+ int32_t *idxarr = alloca (SMALL_STR_SIZE * sizeof (int32_t)); |
715 |
+ unsigned char *rulearr = alloca (SMALL_STR_SIZE + 1); |
716 |
+ |
717 |
+ do |
718 |
+ { |
719 |
+ int32_t tmp = findidx (l_data.table, l_data.indirect, l_data.extra, &cur, |
720 |
+ -1); |
721 |
+ rulearr[idxmax] = tmp >> 24; |
722 |
+ idxarr[idxmax] = tmp & 0xffffff; |
723 |
+ |
724 |
+ ++idxmax; |
725 |
+ } |
726 |
+ while (*cur != L('\0') && idxmax < SMALL_STR_SIZE); |
727 |
+ |
728 |
+ /* This element is only read, the value never used but to determine |
729 |
+ another value which then is ignored. */ |
730 |
+ rulearr[idxmax] = '\0'; |
731 |
+ |
732 |
+ /* Do the transformation. */ |
733 |
+ if (*cur == L('\0')) |
734 |
+ return do_xfrm_cached (dest, n, &l_data, idxmax, idxarr, rulearr); |
735 |
+ else |
736 |
+ return do_xfrm (usrc, dest, n, &l_data); |
737 |
+} |
738 |
libc_hidden_def (STRXFRM) |
739 |
|
740 |
#ifndef WIDE_CHAR_VERSION |
741 |
-- |
742 |
2.3.0 |
743 |
|