Bommels05/Mozilla
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix 71489

Browse Source 
Add johab document encoding support
Currently the convert from johab to unicode part is #if 0 since
we didn't have a way to test it.
r=bstell sr=erik


git-svn-id: svn://10.0.0.236/trunk@197418 18797224-902f-48f8-a5cc-f745e15eee43
This commit is contained in:
ftang%netscape.com
2006-05-17 02:33:20 +00:00
parent 7e3a11639a
commit 89d319d470
2 changed files with 271 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
mozilla/intl/uconv/util/ugen.c
View File

@@ -194,6 +194,24 @@ PRIVATE PRBool uCnGAlways8BytesGLComposedHangul(
PRUint32* outlen
);
PRIVATE PRBool uCheckAndGenJohabHangul(
uShiftTable *shift,
PRInt32* state,
PRUint16 in,
unsigned char* out,
PRUint32 outbuflen,
PRUint32* outlen
);
PRIVATE PRBool uCheckAndGenJohabSymbol(
uShiftTable *shift,
PRInt32* state,
PRUint16 in,
unsigned char* out,
PRUint32 outbuflen,
PRUint32* outlen
);
PRIVATE PRBool uGenComposedHangulCommon(
uShiftTable *shift,
@@ -250,7 +268,9 @@ PRIVATE uGeneratorFunc m_generator[uNumOfCharsetType] =
uCheckAndGen2ByteGRPrefix8EA7,
uCheckAndGenAlways1ByteShiftGL,
uCnGAlways8BytesComposedHangul,
uCnGAlways8BytesGLComposedHangul
uCnGAlways8BytesGLComposedHangul,
uCheckAndGenJohabHangul,
uCheckAndGenJohabSymbol
};
/*=================================================================================
@@ -786,3 +806,118 @@ PRIVATE PRBool uCnGAlways8BytesGLComposedHangul(
{
return uGenComposedHangulCommon(shift,state,in,out,outbuflen,outlen,0x7f);
}
PRIVATE PRBool uCheckAndGenJohabHangul(
uShiftTable *shift,
PRInt32* state,
PRUint16 in,
unsigned char* out,
PRUint32 outbuflen,
PRUint32* outlen
)
{
if(outbuflen < 2)
return PR_FALSE;
else
{
/*
See Table 4-45 (page 183) of CJKV Information Processing
for detail explaination of the following table
*/
/*
static PRUint8 lMap[LCount] = {
2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20
};
Therefore lMap[i] == i+2;
*/
static PRUint8 vMap[VCount] = {
/* no 0,1,2 */
3,4,5,6,7, /* no 8,9 */
10,11,12,13,14,15, /* no 16,17 */
18,19,20,21,22,23, /* no 24,25 */
26,27,28,29
};
static PRUint8 tMap[TCount] = {
1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17, /* no 18 */
19,20,21,22,23,24,25,26,27,28,29
};
PRUint16 SIndex, LIndex, VIndex, TIndex, ch;
/* the following line are copy from Unicode 2.0 page 3-13 */
/* item 1 of Hangul Syllabel Decomposition */
SIndex = in - SBase;
/* the following lines are copy from Unicode 2.0 page 3-14 */
/* item 2 of Hangul Syllabel Decomposition w/ modification */
LIndex = SIndex / NCount;
VIndex = (SIndex % NCount) / TCount;
TIndex = SIndex % TCount;
*outlen = 2;
ch = 0x8000 |
((LIndex+2)<<10) |
(vMap[VIndex]<<5)|
tMap[TIndex];
out[0] = (ch >> 8);
out[1] = ch & 0x00FF;
#if 0
printf("Johab Hangul %x %x in=%x L=%d V=%d T=%d\n", out[0], out[1], in, LIndex, VIndex, TIndex);
#endif
return PR_TRUE;
}
}
PRIVATE PRBool uCheckAndGenJohabSymbol(
uShiftTable *shift,
PRInt32* state,
PRUint16 in,
unsigned char* out,
PRUint32 outbuflen,
PRUint32* outlen
)
{
if(outbuflen < 2)
return PR_FALSE;
else
{
/* The following code are based on the Perl code listed under
* "ISO-2022-KR or EUC-KR to Johab Conversion" (page 1013)
* in the book "CJKV Information Processing" by
* Ken Lunde <lunde@adobe.com>
*
* sub convert2johab($) { # Convert ISO-2022-KR or EUC-KR to Johab
* my @euc = unpack("C*", $_[0]);
* my ($fe_off, $hi_off, $lo_off) = (0,0,1);
* my @out = ();
* while(($hi, $lo) = splice(@euc, 0, 2)) {
* $hi &= 127; $lo &= 127;
* $fe_off = 21 if $hi == 73;
* $fe_off = 34 if $hi == 126;
* ($hi_off, $lo_off) = ($lo_off, $hi_off) if ($hi <74 or $hi >125);
* push(@out, ((($hi+$hi_off) >> 1)+ ($hi <74 ? 200:187)- $fe_off),
* $lo + ((($hi+$lo_off) & 1) ? ($lo > 110 ? 34:16):128));
* }
* return pack("C*", @out);
*/
unsigned char fe_off = 0;
unsigned char hi_off = 0;
unsigned char lo_off = 1;
unsigned char hi = (in >> 8) & 0x7F;
unsigned char lo = in & 0x7F;
if(73 == hi)
fe_off = 21;
if(126 == hi)
fe_off = 34;
if( (hi < 74) || ( hi > 125) )
{
hi_off = 1;
lo_off = 0;
}
*outlen = 2;
out[0] = ((hi+hi_off) >> 1) + ((hi<74) ? 200 : 187 ) - fe_off;
out[1] = lo + (((hi+lo_off) & 1) ? ((lo > 110) ? 34 : 16) : 128);
#if 0
printf("Johab Symbol %x %x in=%x\n", out[0], out[1], in);
#endif
return PR_TRUE;
}
}

136
mozilla/intl/uconv/util/uscan.c
View File

@@ -198,6 +198,22 @@ PRIVATE PRBool uScanComposedHangulCommon(
PRUint32* inscanlen,
PRUint8 mask
);
PRIVATE PRBool uCheckAndScanJohabHangul(
uShiftTable *shift,
PRInt32* state,
unsigned char *in,
PRUint16 *out,
PRUint32 inbuflen,
PRUint32* inscanlen
);
PRIVATE PRBool uCheckAndScanJohabSymbol(
uShiftTable *shift,
PRInt32* state,
unsigned char *in,
PRUint16 *out,
PRUint32 inbuflen,
PRUint32* inscanlen
);
PRIVATE PRBool uScanAlways2Byte(
unsigned char* in,
@@ -243,7 +259,9 @@ PRIVATE uScannerFunc m_scanner[uNumOfCharsetType] =
uCheckAndScan2ByteGRPrefix8EA6,
uCheckAndScan2ByteGRPrefix8EA7,
uCheckAndScanAlways1ByteShiftGL,
uCnSAlways8BytesComposedHangul
uCnSAlways8BytesComposedHangul,
uCheckAndScanJohabHangul,
uCheckAndScanJohabSymbol
};
/*=================================================================================
@@ -784,3 +802,119 @@ PRIVATE PRBool uCnSAlways8BytesGLComposedHangul(
{
return uScanComposedHangulCommon(shift,state,in,out,inbuflen,inscanlen,0x7f);
}
PRIVATE PRBool uCheckAndScanJohabHangul(
uShiftTable *shift,
PRInt32* state,
unsigned char *in,
PRUint16 *out,
PRUint32 inbuflen,
PRUint32* inscanlen
)
{
/* since we don't have code to convert Johab to Unicode right now *
* make this part of code #if 0 to save space untill we fully test it */
#if 0
if(inbuflen < 2)
return PR_FALSE;
else {
/*
* See Table 4-45 Johab Encoding's Five-Bit Binary Patterns in page 183
* of "CJKV Information Processing" for details
*/
static PRUint8 lMap[32]={ /* totaly 19 */
0xff,0xff,0, 1, 2, 3, 4, 5, /* 0-7 */
6, 7, 8, 9, 10, 11, 12, 13, /* 8-15 */
14, 15, 16, 17, 18, 0xff,0xff,0xff, /* 16-23 */
0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff /* 24-31 */
};
static PRUint8 vMap[32]={ /* totaly 21 */
0xff,0xff,0xff,0, 1, 2, 3, 4, /* 0-7 */
0xff,0xff,5, 6, 7, 8, 9, 10, /* 8-15 */
0xff,0xff,11, 12, 13, 14, 15, 16, /* 16-23 */
0xff,0xff,17, 18, 19, 20, 0xff,0xff /* 24-31 */
};
static PRUint8 tMap[32]={ /* totaly 29 */
0xff,0, 1, 2, 3, 4, 5, 6, /* 0-7 */
7, 8, 9, 10, 11, 12, 13, 14, /* 8-15 */
15, 16, 0xff,17, 18, 19, 20, 21, /* 16-23 */
22, 23, 24, 25, 26, 27, 0xff,0xff /* 24-31 */
};
PRUint16 ch = (in[0] << 8) | in[1];
PRUint16 LIndex, VIndex, TIndex;
if(0 == (0x8000 & ch))
return PR_FALSE;
LIndex=lMap[(ch>>10)& 0x1F];
VIndex=vMap[(ch>>5) & 0x1F];
TIndex=tMap[(ch>>0) & 0x1F];
if((0xff==(LIndex)) ||
(0xff==(VIndex)) ||
(0xff==(TIndex)))
return PR_FALSE;
/* the following line is from Unicode 2.0 page 3-13 item 5 */
*out = ( LIndex * VCount + VIndex) * TCount + TIndex + SBase;
return PR_TRUE;
}
#else
return PR_FALSE;
#endif
}
PRIVATE PRBool uCheckAndScanJohabSymbol(
uShiftTable *shift,
PRInt32* state,
unsigned char *in,
PRUint16 *out,
PRUint32 inbuflen,
PRUint32* inscanlen
)
{
/* since we don't have code to convert Johab to Unicode right now
* make this part of code #if 0 to save space untill we fully test it */
#if 0
if(inbuflen < 2)
return PR_FALSE;
else {
/*
* The following code are based on the Perl code lised under
* "Johab to ISO-2022-KR or EUC-KR Conversion" in page 1014 of
* "CJKV Information Processing" by Ken Lunde <lunde@adobe.com>
*
* sub johab2ks ($) { # Convert Johab to ISO-2022-KR
* my @johab = unpack("C*", $_[0]);
* my ($offset, $d8_off) = (0,0);
* my @out = ();
* while(($hi, $lo) = splice($johab, 0, 2)) {
* $offset = 1 if ($hi > 223 and $hi < 250);
* $d8_off = ($hi == 216 and ($lo > 160 ? 94 : 42));
* push (@out, (((($hi - ($hi < 223 ? 200 : 187)) << 1) -
* ($lo < 161 ? 1 : 0) + $offset) + $d8_off),
* $lo - ($lo < 161 ? ($lo > 126 ? 34 : 16) : 128 ));
* }
* return pack ("C*", @out);
* }
* additional comments from Ken Lunde
* $d8_off = ($hi == 216 and ($lo > 160 ? 94 : 42));
* has three possible return values:
* 0 if $hi is not equal to 216
* 94 if $hi is euqal to 216 and if $lo is greater than 160
* 42 if $hi is euqal to 216 and if $lo is not greater than 160
*/
unsigned char hi = in[0];
unsigned char lo = in[1];
PRUint16 offset = (( hi > 223 ) && ( hi < 250)) ? 1 : 0;
PRUint16 d8_off = 0;
if(216 == hi) {
if( lo > 160)
d8_off = 94;
else
d8_off = 42;
}
*out = (((((hi - ((hi < 223) ? 200 : 187)) << 1) -
(lo < 161 ? 1 : 0) + offset) + d8_off) << 8 ) |
(lo - ((lo < 161) ? ((lo > 126) ? 34 : 16) : 128));
return PR_TRUE;
}
#else
return PR_FALSE;
#endif
}