camel-charset-map.c 6.3 KB
Newer Older
Not Zed's avatar
Not Zed committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26

#include <stdio.h>

/*
  if you want to build the charset map, add the root directory of
  libunicode to the include path and define BUILD_MAP,
  then run it as 
    ./a.out > camel-charset-map-private.h

  The tables genereated work like this:

   An indirect array for each page of unicode character
   Each array element has an indirect pointer to one of the bytes of
   the generated bitmask.
*/

#ifdef BUILD_MAP
#include "iso/iso8859-2.h"
#include "iso/iso8859-3.h"
#include "iso/iso8859-4.h"
#include "iso/iso8859-5.h"
#include "iso/iso8859-6.h"
#include "iso/iso8859-7.h"
#include "iso/iso8859-8.h"
#include "iso/iso8859-9.h"
#include "iso/iso8859-10.h"
27
#include "iso/iso8859-13.h"
Not Zed's avatar
Not Zed committed
28 29
#include "iso/iso8859-14.h"
#include "iso/iso8859-15.h"
30 31 32
#include "iso/windows-1250.h"
#include "iso/windows-1252.h"
#include "iso/windows-1257.h"
Not Zed's avatar
Not Zed committed
33 34
#include "iso/koi8-r.h"
#include "iso/koi8-u.h"
35 36 37 38
#include "iso/tis620.2533-1.h"
#include "iso/armscii-8.h"
#include "iso/georgian-academy.h"
#include "iso/georgian-ps.h"
Not Zed's avatar
Not Zed committed
39 40 41 42 43 44 45 46 47 48
#include "msft/cp932.h"
#include "jis/shiftjis.h"

static struct {
	unsigned short *table;
	char *name;
	int type;		/* type of table */
	unsigned int bit;	/* assigned bit */
} tables[] = {
	{ iso8859_2_table, "iso-8859-2", 0, 0} ,
49
	{ iso8859_3_table, "iso-8859-3", 0, 0} ,
Not Zed's avatar
Not Zed committed
50 51 52 53 54 55 56 57
	{ iso8859_4_table, "iso-8859-4", 0, 0},
	{ iso8859_5_table, "iso-8859-5", 0, 0},
/* apparently -6 has special digits? */
	{ iso8859_6_table, "iso-8859-6", 0, 0},
	{ iso8859_7_table, "iso-8859-7", 0, 0},
	{ iso8859_8_table, "iso-8859-8", 0, 0},
	{ iso8859_9_table, "iso-8859-9", 0, 0},
	{ iso8859_10_table, "iso-8859-10", 0, 0},
58
	{ iso8859_13_table, "iso-8859-13", 0, 0},
Not Zed's avatar
Not Zed committed
59 60
	{ iso8859_14_table, "iso-8859-14", 0, 0},
	{ iso8859_15_table, "iso-8859-15", 0, 0},
61 62 63
	{ windows_1250_table, "windows-1250", 0, 0},
	{ windows_1252_table, "windows-1252", 0, 0},
	{ windows_1257_table, "windows-1257", 0, 0},
Not Zed's avatar
Not Zed committed
64 65
	{ koi8_r_table, "koi8-r", 0, 0},
	{ koi8_u_table, "koi8-u", 0, 0},
66 67 68 69
	{ tis_620_table, "tis620.2533-1", 0, 0},
	{ armscii_8_table, "armscii-8", 0, 0},
	{ georgian_academy_table, "georgian-academy", 0, 0},
	{ georgian_ps_table, "georgian-ps", 0, 0},
Not Zed's avatar
Not Zed committed
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
	{ cp932_table, "CP932", 1, 0},
	{ sjis_table, "Shift-JIS", 1, 0},
	{ 0, 0}
};

unsigned int encoding_map[256 * 256];

static void
add_bigmap(unsigned short **table, int bit)
{
	int i;
	int j;

	for (i=0;i<256;i++) {
		unsigned short *tab = table[i];
		if (tab) {
			for (j=0;j<256;j++) {
				if (tab[j])
					encoding_map[tab[j]] |= bit;
			}
		}
	}
}

94
void main(void)
Not Zed's avatar
Not Zed committed
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
{
	int i, j;
	unsigned short *tab;
	int max, min;
	int bit = 0x01;
	int k;
	int bytes;

#if 0
	/* iso-latin-1 (not needed-detected in code) */
	for (i=0;i<256;i++) {
		encoding_map[i] |= bit;
	}
	bit <<= 1;
#endif

	/* dont count the terminator */
	bytes = ((sizeof(tables)/sizeof(tables[0]))+7-1)/8;

	/* the other latin charsets */
	for (j=0;tables[j].table;j++) {
		switch (tables[j].type) {
		case 0:		/* table from 128-256 */
			tab = tables[j].table;
			for (i=0;i<128;i++) {
				/* 0-127 is the common */
				encoding_map[i] |= bit;
				encoding_map[tab[i]] |= bit;
			}
			break;
		case 1:		/* sparse table */
			add_bigmap(tables[j].table, bit);
			break;
		}
		tables[j].bit = bit;
		bit <<= 1;
	}

	printf("/* This file is automatically generated: DO NOT EDIT */\n\n");

	for (i=0;i<256;i++) {
		/* first, do we need this block? */
		for (k=0;k<bytes;k++) {
			for (j=0;j<256;j++) {
				if ((encoding_map[i*256 + j] & (0xff << (k*8))) != 0)
					break;
			}
			if (j < 256) {
				/* yes, dump it */
				printf("static unsigned char m%02x%x[256] = {\n\t", i, k);
				for (j=0;j<256;j++) {
					printf("0x%02x, ", (encoding_map[i*256+j] >> (k*8)) & 0xff );
					if (((j+1)&7) == 0 && j<255)
						printf("\n\t");
				}
				printf("\n};\n\n");
			}
		}
	}

	printf("struct {\n");
	for (k=0;k<bytes;k++) {
		printf("\tunsigned char *bits%d;\n", k);
	}
	printf("} camel_charmap[256] = {\n\t");
	for (i=0;i<256;i++) {
		/* first, do we need this block? */
		printf("{ ");
		for (k=0;k<bytes;k++) {
			for (j=0;j<256;j++) {
				if ((encoding_map[i*256 + j] & (0xff << (k*8))) != 0)
					break;
			}
			if (j < 256) {
				printf("m%02x%x, ", i, k);
			} else {
				printf("0, ");
			}
		}
		printf("}, ");
		if (((i+1)&7) == 0 && i<255)
			printf("\n\t");
	}
	printf("\n};\n\n");

	printf("struct {\n\tconst char *name;\n\tunsigned int bit;\n} camel_charinfo[] = {\n");
	for (j=0;tables[j].table;j++) {
		printf("\t{ \"%s\", 0x%04x },\n", tables[j].name, tables[j].bit);
	}
	printf("};\n\n");

	printf("#define charset_mask(x) \\\n");
	for (k=0;k<bytes;k++) {
		if (k!=0)
			printf("\t| ");
		else
			printf("\t");
		printf("(camel_charmap[(x)>>8].bits%d?camel_charmap[(x)>>8].bits%d[(x)&0xff]<<%d:0)", k, k, k*8);
		if (k<bytes-1)
			printf("\t\\\n");
	}
	printf("\n\n");

}

#else

#include "camel-charset-map.h"
#include "camel-charset-map-private.h"
#include <unicode.h>
#include <glib.h>

207
void camel_charset_init(CamelCharset *c)
Not Zed's avatar
Not Zed committed
208
{
209 210
	c->mask = ~0;
	c->level = 0;
Not Zed's avatar
Not Zed committed
211 212
}

213 214
void
camel_charset_step(CamelCharset *c, const char *in, int len)
Not Zed's avatar
Not Zed committed
215
{
216 217
	register unsigned int mask;
	register int level;
Not Zed's avatar
Not Zed committed
218 219
	const char *inptr = in, *inend = in+len;

220 221 222
	mask = c->mask;
	level = c->level;

Not Zed's avatar
Not Zed committed
223 224 225 226 227 228 229 230 231 232 233
	/* check what charset a given string will fit in */
	while (inptr < inend) {
		unicode_char_t c;
		const char *newinptr;
		newinptr = unicode_get_utf8(inptr, &c);
		if (newinptr == NULL) {
			inptr++;
			continue;
		}
		inptr = newinptr;
		if (c<=0xffff) {
Not Zed's avatar
Not Zed committed
234
			mask &= charset_mask(c);
Not Zed's avatar
Not Zed committed
235 236 237 238 239 240 241 242 243 244 245
		
			if (c>=128 && c<256)
				level = MAX(level, 1);
			else if (c>=256)
				level = MAX(level, 2);
		} else {
			mask = 0;
			level = MAX(level, 2);
		}
	}

246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
	c->mask = mask;
	c->level = level;
}

/* gets the best charset from the mask of chars in it */
static const char *
camel_charset_best_mask(unsigned int mask)
{
	int i;

	for (i=0;i<sizeof(camel_charinfo)/sizeof(camel_charinfo[0]);i++) {
		if (camel_charinfo[i].bit & mask)
			return camel_charinfo[i].name;
	}
	return "UTF-8";
}

const char *camel_charset_best_name(CamelCharset *charset)
{
	if (charset->level == 1)
Not Zed's avatar
Not Zed committed
266
		return "ISO-8859-1";
267 268
	else if (charset->level == 2)
		return camel_charset_best_mask(charset->mask);
269 270
	else
		return NULL;
271 272 273 274 275 276 277 278 279 280 281 282

}

/* finds the minimum charset for this string NULL means US-ASCII */
const char *
camel_charset_best(const char *in, int len)
{
	CamelCharset charset;

	camel_charset_init(&charset);
	camel_charset_step(&charset, in, len);
	return camel_charset_best_name(&charset);
Not Zed's avatar
Not Zed committed
283 284 285 286 287
}


#endif /* !BUILD_MAP */