Author: br Date: Tue Nov 1 13:54:44 2016 New Revision: 308170 URL: https://svnweb.freebsd.org/changeset/base/308170
Log: Locale fix for endian big (EB) machines. We have locale files generated on EL machines (e.g. during cross-build on amd64 host), but then we are using them on EB machines (e.g. MIPS64EB), so proceed byte-swap if necessary. All the libc tests passed successfully, including Russian collation. Tested by: br@, Hongyan Xia <hx...@cam.ac.uk> Sponsored by: DARPA, AFRL Sponsored by: HEIF5 Differential Revision: https://reviews.freebsd.org/D8281 Added: head/lib/libc/locale/endian.h (contents, props changed) Modified: head/lib/libc/locale/collate.c head/lib/libc/locale/rune.c Modified: head/lib/libc/locale/collate.c ============================================================================== --- head/lib/libc/locale/collate.c Tue Nov 1 12:47:19 2016 (r308169) +++ head/lib/libc/locale/collate.c Tue Nov 1 13:54:44 2016 (r308170) @@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$"); #include <fcntl.h> #include "un-namespace.h" +#include "endian.h" #include "collate.h" #include "setlocale.h" #include "ldpart.h" @@ -160,7 +161,7 @@ __collate_load_tables_l(const char *enco if ((info->directive_count < 1) || (info->directive_count >= COLL_WEIGHTS_MAX) || - ((chains = info->chain_count) < 0)) { + ((chains = BSWAP(info->chain_count)) < 0)) { (void) munmap(map, sbuf.st_size); errno = EINVAL; return (_LDP_ERROR); @@ -168,9 +169,9 @@ __collate_load_tables_l(const char *enco i = (sizeof (collate_char_t) * (UCHAR_MAX + 1)) + (sizeof (collate_chain_t) * chains) + - (sizeof (collate_large_t) * info->large_count); + (sizeof (collate_large_t) * BSWAP(info->large_count)); for (z = 0; z < info->directive_count; z++) { - i += sizeof (collate_subst_t) * info->subst_count[z]; + i += sizeof (collate_subst_t) * BSWAP(info->subst_count[z]); } if (i != (sbuf.st_size - (TMP - map))) { (void) munmap(map, sbuf.st_size); @@ -183,9 +184,9 @@ __collate_load_tables_l(const char *enco TMP += sizeof (collate_char_t) * (UCHAR_MAX + 1); for (z = 0; z < info->directive_count; z++) { - if (info->subst_count[z] > 0) { + if (BSWAP(info->subst_count[z]) > 0) { table->subst_table[z] = (void *)TMP; - TMP += info->subst_count[z] * sizeof (collate_subst_t); + TMP += BSWAP(info->subst_count[z]) * sizeof (collate_subst_t); } else { table->subst_table[z] = NULL; } @@ -196,7 +197,7 @@ __collate_load_tables_l(const char *enco TMP += chains * sizeof (collate_chain_t); } else table->chain_pri_table = NULL; - if (info->large_count > 0) + if (BSWAP(info->large_count) > 0) table->large_pri_table = (void *)TMP; else table->large_pri_table = NULL; @@ -209,7 +210,7 @@ static const int32_t * substsearch(struct xlocale_collate *table, const wchar_t key, int pass) { const collate_subst_t *p; - int n = table->info->subst_count[pass]; + int n = BSWAP(table->info->subst_count[pass]); if (n == 0) return (NULL); @@ -221,7 +222,8 @@ substsearch(struct xlocale_collate *tabl return (NULL); p = table->subst_table[pass] + (key & ~COLLATE_SUBST_PRIORITY); - assert(p->key == key); + assert(BSWAP(p->key) == key); + return (p->pri); } @@ -229,7 +231,7 @@ static collate_chain_t * chainsearch(struct xlocale_collate *table, const wchar_t *key, int *len) { int low = 0; - int high = table->info->chain_count - 1;; + int high = BSWAP(table->info->chain_count) - 1; int next, compar, l; collate_chain_t *p; collate_chain_t *tab = table->chain_pri_table; @@ -240,7 +242,7 @@ chainsearch(struct xlocale_collate *tabl while (low <= high) { next = (low + high) / 2; p = tab + next; - compar = *key - *p->str; + compar = *key - le16toh(*p->str); if (compar == 0) { l = wcsnlen(p->str, COLLATE_STR_LEN); compar = wcsncmp(key, p->str, l); @@ -261,7 +263,7 @@ static collate_large_t * largesearch(struct xlocale_collate *table, const wchar_t key) { int low = 0; - int high = table->info->large_count - 1; + int high = BSWAP(table->info->large_count) - 1; int next, compar; collate_large_t *p; collate_large_t *tab = table->large_pri_table; @@ -272,7 +274,7 @@ largesearch(struct xlocale_collate *tabl while (low <= high) { next = (low + high) / 2; p = tab + next; - compar = key - p->val; + compar = key - BSWAP(p->val); if (compar == 0) return (p); if (compar > 0) @@ -337,15 +339,15 @@ _collate_lookup(struct xlocale_collate * * Character is a small (8-bit) character. * We just look these up directly for speed. */ - *pri = table->char_pri_table[*t].pri[which]; + *pri = BSWAP(table->char_pri_table[*t].pri[which]); - } else if ((table->info->large_count > 0) && + } else if ((BSWAP(table->info->large_count) > 0) && ((match = largesearch(table, *t)) != NULL)) { /* * Character was found in the extended table. */ - *pri = match->pri.pri[which]; + *pri = BSWAP(match->pri.pri[which]); } else { /* @@ -355,7 +357,7 @@ _collate_lookup(struct xlocale_collate * /* Mask off sign bit to prevent ordering confusion. */ *pri = (*t & COLLATE_MAX_PRIORITY); } else { - *pri = table->info->undef_pri[which]; + *pri = BSWAP(table->info->undef_pri[which]); } /* No substitutions for undefined characters! */ return; @@ -374,9 +376,9 @@ _collate_lookup(struct xlocale_collate * * code ensures this for us. */ if ((sptr = substsearch(table, *pri, which)) != NULL) { - if ((*pri = *sptr) > 0) { + if ((*pri = BSWAP(*sptr)) > 0) { sptr++; - *state = *sptr ? sptr : NULL; + *state = BSWAP(*sptr) ? sptr : NULL; } } @@ -518,7 +520,7 @@ static int xfrm(struct xlocale_collate *table, unsigned char *p, int pri, int pass) { /* we use unsigned to ensure zero fill on right shift */ - uint32_t val = (uint32_t)table->info->pri_count[pass]; + uint32_t val = BSWAP((uint32_t)table->info->pri_count[pass]); int nc = 0; while (val) { @@ -678,7 +680,7 @@ __collate_equiv_value(locale_t locale, c e = -1; if (*str <= UCHAR_MAX) e = table->char_pri_table[*str].pri[0]; - else if (table->info->large_count > 0) { + else if (BSWAP(table->info->large_count) > 0) { collate_large_t *match_large; match_large = largesearch(table, *str); if (match_large) @@ -688,7 +690,7 @@ __collate_equiv_value(locale_t locale, c return (1); return (e > 0 ? e : 0); } - if (table->info->chain_count > 0) { + if (BSWAP(table->info->chain_count) > 0) { wchar_t name[COLLATE_STR_LEN]; collate_chain_t *match_chain; int clen; Added: head/lib/libc/locale/endian.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/lib/libc/locale/endian.h Tue Nov 1 13:54:44 2016 (r308170) @@ -0,0 +1,52 @@ +/*- + * Copyright (c) 2016 Ruslan Bukin <b...@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include <sys/endian.h> + +/* + * We assume locale files were generated on EL machine + * (e.g. during cross build on amd64 host), but used on EB + * machine (e.g. MIPS64EB), so convert it to host endianness. + * + * TODO: detect host endianness on the build machine and use + * correct macros here. + */ + +#if BYTE_ORDER == BIG_ENDIAN && defined(__mips__) +#define BSWAP(x) le32toh(x) +#else +#define BSWAP(x) x +#endif Modified: head/lib/libc/locale/rune.c ============================================================================== --- head/lib/libc/locale/rune.c Tue Nov 1 12:47:19 2016 (r308169) +++ head/lib/libc/locale/rune.c Tue Nov 1 13:54:44 2016 (r308170) @@ -52,6 +52,7 @@ __FBSDID("$FreeBSD$"); #include <unistd.h> #include "un-namespace.h" +#include "endian.h" #include "runefile.h" _RuneLocale * @@ -107,29 +108,29 @@ _Read_RuneMagi(const char *fname) } runetype_ext_ranges = (_FileRuneEntry *)variable; - variable = runetype_ext_ranges + frl->runetype_ext_nranges; + variable = runetype_ext_ranges + BSWAP(frl->runetype_ext_nranges); if (variable > lastp) { goto invalid; } maplower_ext_ranges = (_FileRuneEntry *)variable; - variable = maplower_ext_ranges + frl->maplower_ext_nranges; + variable = maplower_ext_ranges + BSWAP(frl->maplower_ext_nranges); if (variable > lastp) { goto invalid; } mapupper_ext_ranges = (_FileRuneEntry *)variable; - variable = mapupper_ext_ranges + frl->mapupper_ext_nranges; + variable = mapupper_ext_ranges + BSWAP(frl->mapupper_ext_nranges); if (variable > lastp) { goto invalid; } frr = runetype_ext_ranges; - for (x = 0; x < frl->runetype_ext_nranges; ++x) { + for (x = 0; x < BSWAP(frl->runetype_ext_nranges); ++x) { uint32_t *types; - if (frr[x].map == 0) { - int len = frr[x].max - frr[x].min + 1; + if (BSWAP(frr[x].map) == 0) { + int len = BSWAP(frr[x].max) - BSWAP(frr[x].min) + 1; types = variable; variable = types + len; runetype_ext_len += len; @@ -139,7 +140,7 @@ _Read_RuneMagi(const char *fname) } } - if ((char *)variable + frl->variable_len > (char *)lastp) { + if ((char *)variable + BSWAP(frl->variable_len) > (char *)lastp) { goto invalid; } @@ -147,10 +148,10 @@ _Read_RuneMagi(const char *fname) * Convert from disk format to host format. */ data = malloc(sizeof(_RuneLocale) + - (frl->runetype_ext_nranges + frl->maplower_ext_nranges + - frl->mapupper_ext_nranges) * sizeof(_RuneEntry) + + (BSWAP(frl->runetype_ext_nranges) + BSWAP(frl->maplower_ext_nranges) + + BSWAP(frl->mapupper_ext_nranges)) * sizeof(_RuneEntry) + runetype_ext_len * sizeof(*rr->__types) + - frl->variable_len); + BSWAP(frl->variable_len)); if (data == NULL) { saverr = errno; munmap(fdata, sb.st_size); @@ -164,15 +165,15 @@ _Read_RuneMagi(const char *fname) memcpy(rl->__magic, _RUNE_MAGIC_1, sizeof(rl->__magic)); memcpy(rl->__encoding, frl->encoding, sizeof(rl->__encoding)); - rl->__variable_len = frl->variable_len; - rl->__runetype_ext.__nranges = frl->runetype_ext_nranges; - rl->__maplower_ext.__nranges = frl->maplower_ext_nranges; - rl->__mapupper_ext.__nranges = frl->mapupper_ext_nranges; + rl->__variable_len = BSWAP(frl->variable_len); + rl->__runetype_ext.__nranges = BSWAP(frl->runetype_ext_nranges); + rl->__maplower_ext.__nranges = BSWAP(frl->maplower_ext_nranges); + rl->__mapupper_ext.__nranges = BSWAP(frl->mapupper_ext_nranges); for (x = 0; x < _CACHED_RUNES; ++x) { - rl->__runetype[x] = frl->runetype[x]; - rl->__maplower[x] = frl->maplower[x]; - rl->__mapupper[x] = frl->mapupper[x]; + rl->__runetype[x] = BSWAP(frl->runetype[x]); + rl->__maplower[x] = BSWAP(frl->maplower[x]); + rl->__mapupper[x] = BSWAP(frl->mapupper[x]); } rl->__runetype_ext.__ranges = (_RuneEntry *)rl->__variable; @@ -187,15 +188,15 @@ _Read_RuneMagi(const char *fname) rl->__variable = rl->__mapupper_ext.__ranges + rl->__mapupper_ext.__nranges; - variable = mapupper_ext_ranges + frl->mapupper_ext_nranges; + variable = mapupper_ext_ranges + BSWAP(frl->mapupper_ext_nranges); frr = runetype_ext_ranges; rr = rl->__runetype_ext.__ranges; for (x = 0; x < rl->__runetype_ext.__nranges; ++x) { uint32_t *types; - rr[x].__min = frr[x].min; - rr[x].__max = frr[x].max; - rr[x].__map = frr[x].map; + rr[x].__min = BSWAP(frr[x].min); + rr[x].__max = BSWAP(frr[x].max); + rr[x].__map = BSWAP(frr[x].map); if (rr[x].__map == 0) { int len = rr[x].__max - rr[x].__min + 1; types = variable; @@ -211,17 +212,17 @@ _Read_RuneMagi(const char *fname) frr = maplower_ext_ranges; rr = rl->__maplower_ext.__ranges; for (x = 0; x < rl->__maplower_ext.__nranges; ++x) { - rr[x].__min = frr[x].min; - rr[x].__max = frr[x].max; - rr[x].__map = frr[x].map; + rr[x].__min = BSWAP(frr[x].min); + rr[x].__max = BSWAP(frr[x].max); + rr[x].__map = BSWAP(frr[x].map); } frr = mapupper_ext_ranges; rr = rl->__mapupper_ext.__ranges; for (x = 0; x < rl->__mapupper_ext.__nranges; ++x) { - rr[x].__min = frr[x].min; - rr[x].__max = frr[x].max; - rr[x].__map = frr[x].map; + rr[x].__min = BSWAP(frr[x].min); + rr[x].__max = BSWAP(frr[x].max); + rr[x].__map = BSWAP(frr[x].map); } memcpy(rl->__variable, variable, rl->__variable_len); _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
