gconvert.c 56.2 KB
Newer Older
1 2 3 4
/* GLIB - Library of useful routines for C programming
 *
 * gconvert.c: Convert between character sets using iconv
 * Copyright Red Hat Inc., 2000
5
 * Authors: Havoc Pennington <hp@redhat.com>, Owen Taylor <otaylor@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 */

23
#include "config.h"
Owen Taylor's avatar
Owen Taylor committed
24

25 26 27
#include "glib.h"

#ifndef G_OS_WIN32
28
#include <iconv.h>
29
#endif
30
#include <errno.h>
31
#include <stdio.h>
32
#include <string.h>
33 34
#include <stdlib.h>

35
#include "gprintfint.h"
36
#include "gthreadprivate.h"
37
#include "gunicode.h"
Tor Lillqvist's avatar
Tor Lillqvist committed
38

39 40 41 42
#ifdef G_OS_WIN32
#include "win_iconv.c"
#endif

43 44
#ifdef G_PLATFORM_WIN32
#define STRICT
45
#include <windows.h>
46
#undef STRICT
47
#endif
48

Owen Taylor's avatar
Owen Taylor committed
49
#include "glibintl.h"
50

51 52
#if defined(USE_LIBICONV_GNU) && !defined (_LIBICONV_H)
#error GNU libiconv in use but included iconv.h not from libiconv
53
#endif
54 55
#if !defined(USE_LIBICONV_GNU) && defined (_LIBICONV_H)
#error GNU libiconv not in use but included iconv.h is from libiconv
56 57
#endif

58

59 60 61 62 63 64
/* We try to terminate strings in unknown charsets with this many zero bytes
 * to ensure that multibyte strings really are nul-terminated when we return
 * them from g_convert() and friends.
 */
#define NUL_TERMINATOR_LENGTH 4

65
GQuark 
66
g_convert_error_quark (void)
67
{
68
  return g_quark_from_static_string ("g_convert_error");
69 70
}

71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
static gboolean
try_conversion (const char *to_codeset,
		const char *from_codeset,
		iconv_t    *cd)
{
  *cd = iconv_open (to_codeset, from_codeset);

  if (*cd == (iconv_t)-1 && errno == EINVAL)
    return FALSE;
  else
    return TRUE;
}

static gboolean
try_to_aliases (const char **to_aliases,
		const char  *from_codeset,
		iconv_t     *cd)
{
  if (to_aliases)
    {
      const char **p = to_aliases;
      while (*p)
	{
	  if (try_conversion (*p, from_codeset, cd))
	    return TRUE;

	  p++;
	}
    }

  return FALSE;
}

104 105
G_GNUC_INTERNAL extern const char ** 
_g_charset_get_aliases (const char *canonical_name);
106

Havoc Pennington's avatar
docs  
Havoc Pennington committed
107 108 109 110 111
/**
 * g_iconv_open:
 * @to_codeset: destination codeset
 * @from_codeset: source codeset
 * 
112
 * Same as the standard UNIX routine iconv_open(), but
113
 * may be implemented via libiconv on UNIX flavors that lack
Havoc Pennington's avatar
docs  
Havoc Pennington committed
114 115
 * a native implementation.
 * 
Owen Taylor's avatar
Owen Taylor committed
116
 * GLib provides g_convert() and g_locale_to_utf8() which are likely
Havoc Pennington's avatar
docs  
Havoc Pennington committed
117 118
 * more convenient than the raw iconv wrappers.
 * 
119 120
 * Return value: a "conversion descriptor", or (GIConv)-1 if
 *  opening the converter failed.
Havoc Pennington's avatar
docs  
Havoc Pennington committed
121
 **/
122 123 124 125
GIConv
g_iconv_open (const gchar  *to_codeset,
	      const gchar  *from_codeset)
{
126
  iconv_t cd;
127
  
128 129 130
  if (!try_conversion (to_codeset, from_codeset, &cd))
    {
      const char **to_aliases = _g_charset_get_aliases (to_codeset);
131
      const char **from_aliases = _g_charset_get_aliases (from_codeset);
132 133 134 135 136 137 138

      if (from_aliases)
	{
	  const char **p = from_aliases;
	  while (*p)
	    {
	      if (try_conversion (to_codeset, *p, &cd))
139
		goto out;
140 141

	      if (try_to_aliases (to_aliases, *p, &cd))
142
		goto out;
143 144 145 146 147 148

	      p++;
	    }
	}

      if (try_to_aliases (to_aliases, from_codeset, &cd))
149
	goto out;
150 151
    }

152
 out:
153
  return (cd == (iconv_t)-1) ? (GIConv)-1 : (GIConv)cd;
154 155
}

Havoc Pennington's avatar
docs  
Havoc Pennington committed
156 157 158 159 160 161 162 163
/**
 * g_iconv:
 * @converter: conversion descriptor from g_iconv_open()
 * @inbuf: bytes to convert
 * @inbytes_left: inout parameter, bytes remaining to convert in @inbuf
 * @outbuf: converted output bytes
 * @outbytes_left: inout parameter, bytes available to fill in @outbuf
 * 
164
 * Same as the standard UNIX routine iconv(), but
165
 * may be implemented via libiconv on UNIX flavors that lack
Havoc Pennington's avatar
docs  
Havoc Pennington committed
166 167
 * a native implementation.
 *
Owen Taylor's avatar
Owen Taylor committed
168
 * GLib provides g_convert() and g_locale_to_utf8() which are likely
Havoc Pennington's avatar
docs  
Havoc Pennington committed
169 170 171 172
 * more convenient than the raw iconv wrappers.
 * 
 * Return value: count of non-reversible conversions, or -1 on error
 **/
173
gsize 
174 175
g_iconv (GIConv   converter,
	 gchar  **inbuf,
176
	 gsize   *inbytes_left,
177
	 gchar  **outbuf,
178
	 gsize   *outbytes_left)
179 180 181 182 183 184
{
  iconv_t cd = (iconv_t)converter;

  return iconv (cd, inbuf, inbytes_left, outbuf, outbytes_left);
}

Havoc Pennington's avatar
docs  
Havoc Pennington committed
185 186 187 188
/**
 * g_iconv_close:
 * @converter: a conversion descriptor from g_iconv_open()
 *
189
 * Same as the standard UNIX routine iconv_close(), but
190
 * may be implemented via libiconv on UNIX flavors that lack
Havoc Pennington's avatar
docs  
Havoc Pennington committed
191
 * a native implementation. Should be called to clean up
Matthias Clasen's avatar
Matthias Clasen committed
192
 * the conversion descriptor from g_iconv_open() when
Havoc Pennington's avatar
docs  
Havoc Pennington committed
193 194
 * you are done converting things.
 *
Owen Taylor's avatar
Owen Taylor committed
195
 * GLib provides g_convert() and g_locale_to_utf8() which are likely
Havoc Pennington's avatar
docs  
Havoc Pennington committed
196 197 198 199
 * more convenient than the raw iconv wrappers.
 * 
 * Return value: -1 on error, 0 on success
 **/
200 201 202 203 204 205 206 207
gint
g_iconv_close (GIConv converter)
{
  iconv_t cd = (iconv_t)converter;

  return iconv_close (cd);
}

208

209 210
#ifdef NEED_ICONV_CACHE

211 212 213 214 215 216
#define ICONV_CACHE_SIZE   (16)

struct _iconv_cache_bucket {
  gchar *key;
  guint32 refcount;
  gboolean used;
217
  GIConv cd;
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
};

static GList *iconv_cache_list;
static GHashTable *iconv_cache;
static GHashTable *iconv_open_hash;
static guint iconv_cache_size = 0;
G_LOCK_DEFINE_STATIC (iconv_cache_lock);

/* caller *must* hold the iconv_cache_lock */
static void
iconv_cache_init (void)
{
  static gboolean initialized = FALSE;
  
  if (initialized)
    return;
  
  iconv_cache_list = NULL;
  iconv_cache = g_hash_table_new (g_str_hash, g_str_equal);
  iconv_open_hash = g_hash_table_new (g_direct_hash, g_direct_equal);
  
  initialized = TRUE;
}


243
/*
244 245 246 247 248 249 250
 * iconv_cache_bucket_new:
 * @key: cache key
 * @cd: iconv descriptor
 *
 * Creates a new cache bucket, inserts it into the cache and
 * increments the cache size.
 *
251 252
 * This assumes ownership of @key.
 *
253 254
 * Returns a pointer to the newly allocated cache bucket.
 **/
255
static struct _iconv_cache_bucket *
256
iconv_cache_bucket_new (gchar *key, GIConv cd)
257 258 259 260
{
  struct _iconv_cache_bucket *bucket;
  
  bucket = g_new (struct _iconv_cache_bucket, 1);
261
  bucket->key = key;
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
  bucket->refcount = 1;
  bucket->used = TRUE;
  bucket->cd = cd;
  
  g_hash_table_insert (iconv_cache, bucket->key, bucket);
  
  /* FIXME: if we sorted the list so items with few refcounts were
     first, then we could expire them faster in iconv_cache_expire_unused () */
  iconv_cache_list = g_list_prepend (iconv_cache_list, bucket);
  
  iconv_cache_size++;
  
  return bucket;
}


278
/*
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
 * iconv_cache_bucket_expire:
 * @node: cache bucket's node
 * @bucket: cache bucket
 *
 * Expires a single cache bucket @bucket. This should only ever be
 * called on a bucket that currently has no used iconv descriptors
 * open.
 *
 * @node is not a required argument. If @node is not supplied, we
 * search for it ourselves.
 **/
static void
iconv_cache_bucket_expire (GList *node, struct _iconv_cache_bucket *bucket)
{
  g_hash_table_remove (iconv_cache, bucket->key);
  
  if (node == NULL)
    node = g_list_find (iconv_cache_list, bucket);
  
  g_assert (node != NULL);
  
  if (node->prev)
    {
      node->prev->next = node->next;
      if (node->next)
        node->next->prev = node->prev;
    }
  else
    {
      iconv_cache_list = node->next;
      if (node->next)
        node->next->prev = NULL;
    }
  
  g_list_free_1 (node);
  
  g_free (bucket->key);
  g_iconv_close (bucket->cd);
  g_free (bucket);
  
  iconv_cache_size--;
}


323
/*
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
 * iconv_cache_expire_unused:
 *
 * Expires as many unused cache buckets as it needs to in order to get
 * the total number of buckets < ICONV_CACHE_SIZE.
 **/
static void
iconv_cache_expire_unused (void)
{
  struct _iconv_cache_bucket *bucket;
  GList *node, *next;
  
  node = iconv_cache_list;
  while (node && iconv_cache_size >= ICONV_CACHE_SIZE)
    {
      next = node->next;
      
      bucket = node->data;
      if (bucket->refcount == 0)
        iconv_cache_bucket_expire (node, bucket);
      
      node = next;
    }
}

Havoc Pennington's avatar
Havoc Pennington committed
348
static GIConv
349
open_converter (const gchar *to_codeset,
350
		const gchar *from_codeset,
351 352
		GError     **error)
{
353
  struct _iconv_cache_bucket *bucket;
354
  gchar *key, *dyn_key, auto_key[80];
355
  GIConv cd;
356
  gsize len_from_codeset, len_to_codeset;
357 358
  
  /* create our key */
359 360 361 362 363 364 365 366 367 368 369 370 371
  len_from_codeset = strlen (from_codeset);
  len_to_codeset = strlen (to_codeset);
  if (len_from_codeset + len_to_codeset + 2 < sizeof (auto_key))
    {
      key = auto_key;
      dyn_key = NULL;
    }
  else
    key = dyn_key = g_malloc (len_from_codeset + len_to_codeset + 2);
  memcpy (key, from_codeset, len_from_codeset);
  key[len_from_codeset] = ':';
  strcpy (key + len_from_codeset + 1, to_codeset);

372 373 374 375 376 377 378
  G_LOCK (iconv_cache_lock);
  
  /* make sure the cache has been initialized */
  iconv_cache_init ();
  
  bucket = g_hash_table_lookup (iconv_cache, key);
  if (bucket)
379
    {
380 381
      g_free (dyn_key);

382 383 384
      if (bucket->used)
        {
          cd = g_iconv_open (to_codeset, from_codeset);
385
          if (cd == (GIConv) -1)
386 387
            goto error;
        }
388
      else
389
        {
390 391 392 393
	  /* Apparently iconv on Solaris <= 7 segfaults if you pass in
	   * NULL for anything but inbuf; work around that. (NULL outbuf
	   * or NULL *outbuf is allowed by Unix98.)
	   */
394
	  gsize inbytes_left = 0;
395
	  gchar *outbuf = NULL;
396
	  gsize outbytes_left = 0;
397
		
398 399 400 401
          cd = bucket->cd;
          bucket->used = TRUE;
          
          /* reset the descriptor */
402
          g_iconv (cd, NULL, &inbytes_left, &outbuf, &outbytes_left);
403 404 405
        }
      
      bucket->refcount++;
406
    }
407 408 409
  else
    {
      cd = g_iconv_open (to_codeset, from_codeset);
410 411 412 413 414
      if (cd == (GIConv) -1) 
	{
	  g_free (dyn_key);
	  goto error;
	}
415 416
      
      iconv_cache_expire_unused ();
417 418

      bucket = iconv_cache_bucket_new (dyn_key ? dyn_key : g_strdup (key), cd);
419 420 421 422 423 424 425 426 427 428 429 430 431
    }
  
  g_hash_table_insert (iconv_open_hash, cd, bucket->key);
  
  G_UNLOCK (iconv_cache_lock);
  
  return cd;
  
 error:
  
  G_UNLOCK (iconv_cache_lock);
  
  /* Something went wrong.  */
432 433 434 435 436 437 438 439 440 441 442
  if (error)
    {
      if (errno == EINVAL)
	g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_NO_CONVERSION,
		     _("Conversion from character set '%s' to '%s' is not supported"),
		     from_codeset, to_codeset);
      else
	g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
		     _("Could not open converter from '%s' to '%s'"),
		     from_codeset, to_codeset);
    }
443
  
444
  return cd;
445
}
446

447 448 449 450 451
static int
close_converter (GIConv converter)
{
  struct _iconv_cache_bucket *bucket;
  const gchar *key;
452
  GIConv cd;
453
  
454
  cd = converter;
455
  
456
  if (cd == (GIConv) -1)
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493
    return 0;
  
  G_LOCK (iconv_cache_lock);
  
  key = g_hash_table_lookup (iconv_open_hash, cd);
  if (key)
    {
      g_hash_table_remove (iconv_open_hash, cd);
      
      bucket = g_hash_table_lookup (iconv_cache, key);
      g_assert (bucket);
      
      bucket->refcount--;
      
      if (cd == bucket->cd)
        bucket->used = FALSE;
      else
        g_iconv_close (cd);
      
      if (!bucket->refcount && iconv_cache_size > ICONV_CACHE_SIZE)
        {
          /* expire this cache bucket */
          iconv_cache_bucket_expire (NULL, bucket);
        }
    }
  else
    {
      G_UNLOCK (iconv_cache_lock);
      
      g_warning ("This iconv context wasn't opened using open_converter");
      
      return g_iconv_close (converter);
    }
  
  G_UNLOCK (iconv_cache_lock);
  
  return 0;
494 495
}

496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
#else  /* !NEED_ICONV_CACHE */

static GIConv
open_converter (const gchar *to_codeset,
		const gchar *from_codeset,
		GError     **error)
{
  GIConv cd;

  cd = g_iconv_open (to_codeset, from_codeset);

  if (cd == (GIConv) -1)
    {
      /* Something went wrong.  */
      if (error)
	{
	  if (errno == EINVAL)
	    g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_NO_CONVERSION,
			 _("Conversion from character set '%s' to '%s' is not supported"),
			 from_codeset, to_codeset);
	  else
	    g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
			 _("Could not open converter from '%s' to '%s'"),
			 from_codeset, to_codeset);
	}
    }
  
  return cd;
}

static int
close_converter (GIConv cd)
{
  if (cd == (GIConv) -1)
    return 0;
  
  return g_iconv_close (cd);  
}

#endif /* NEED_ICONV_CACHE */

537 538 539
/**
 * g_convert_with_iconv:
 * @str:           the string to convert
540 541
 * @len:           the length of the string, or -1 if the string is 
 *                 nul-terminated<footnoteref linkend="nul-unsafe"/>. 
542 543 544
 * @converter:     conversion descriptor from g_iconv_open()
 * @bytes_read:    location to store the number of bytes in the
 *                 input string that were successfully converted, or %NULL.
Matthias Clasen's avatar
Matthias Clasen committed
545
 *                 Even if the conversion was successful, this may be 
546
 *                 less than @len if there were partial characters
547
 *                 at the end of the input. If the error
548 549
 *                 #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
 *                 stored will the byte offset after the last valid
550
 *                 input sequence.
551
 * @bytes_written: the number of bytes stored in the output buffer (not 
Matthias Clasen's avatar
Matthias Clasen committed
552
 *                 including the terminating nul).
553 554 555
 * @error:         location to store the error occuring, or %NULL to ignore
 *                 errors. Any of the errors in #GConvertError may occur.
 *
556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571
 * Converts a string from one character set to another. 
 * 
 * Note that you should use g_iconv() for streaming 
 * conversions<footnote id="streaming-state">
 *  <para>
 * Despite the fact that @byes_read can return information about partial 
 * characters, the <literal>g_convert_...</literal> functions
 * are not generally suitable for streaming. If the underlying converter 
 * being used maintains internal state, then this won't be preserved 
 * across successive calls to g_convert(), g_convert_with_iconv() or 
 * g_convert_with_fallback(). (An example of this is the GNU C converter 
 * for CP1255 which does not emit a base character until it knows that 
 * the next character is not a mark that could combine with the base 
 * character.)
 *  </para>
 * </footnote>. 
572 573
 *
 * Return value: If the conversion was successful, a newly allocated
574 575
 *               nul-terminated string, which must be freed with
 *               g_free(). Otherwise %NULL and @error will be set.
576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
 **/
gchar*
g_convert_with_iconv (const gchar *str,
		      gssize       len,
		      GIConv       converter,
		      gsize       *bytes_read, 
		      gsize       *bytes_written, 
		      GError     **error)
{
  gchar *dest;
  gchar *outp;
  const gchar *p;
  gsize inbytes_remaining;
  gsize outbytes_remaining;
  gsize err;
  gsize outbuf_size;
  gboolean have_error = FALSE;
593
  gboolean done = FALSE;
594
  gboolean reset = FALSE;
595 596 597
  
  g_return_val_if_fail (converter != (GIConv) -1, NULL);
     
598 599 600 601 602
  if (len < 0)
    len = strlen (str);

  p = str;
  inbytes_remaining = len;
603
  outbuf_size = len + NUL_TERMINATOR_LENGTH;
604
  
605
  outbytes_remaining = outbuf_size - NUL_TERMINATOR_LENGTH;
606 607
  outp = dest = g_malloc (outbuf_size);

608
  while (!done && !have_error)
609
    {
610 611 612 613
      if (reset)
        err = g_iconv (converter, NULL, &inbytes_remaining, &outp, &outbytes_remaining);
      else
        err = g_iconv (converter, (char **)&p, &inbytes_remaining, &outp, &outbytes_remaining);
614

615
      if (err == (gsize) -1)
616
	{
617 618 619 620
	  switch (errno)
	    {
	    case EINVAL:
	      /* Incomplete text, do not report an error */
621
	      done = TRUE;
622 623 624
	      break;
	    case E2BIG:
	      {
625
		gsize used = outp - dest;
626
		
627 628 629 630
		outbuf_size *= 2;
		dest = g_realloc (dest, outbuf_size);
		
		outp = dest + used;
631
		outbytes_remaining = outbuf_size - used - NUL_TERMINATOR_LENGTH;
632 633 634 635
	      }
	      break;
	    case EILSEQ:
	      if (error)
636 637
		g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
                                     _("Invalid byte sequence in conversion input"));
638 639 640
	      have_error = TRUE;
	      break;
	    default:
Christian Persch's avatar
Christian Persch committed
641 642 643 644 645 646 647
              {
                int errsv = errno;
                
                g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
                             _("Error during conversion: %s"),
                             g_strerror (errsv));
              }
648 649 650 651 652 653
	      have_error = TRUE;
	      break;
	    }
	}
      else 
	{
654
	  if (!reset)
655 656
	    {
	      /* call g_iconv with NULL inbuf to cleanup shift state */
657
	      reset = TRUE;
658 659 660 661
	      inbytes_remaining = 0;
	    }
	  else
	    done = TRUE;
662 663 664
	}
    }

665
  memset (outp, 0, NUL_TERMINATOR_LENGTH);
666 667 668
  
  if (bytes_read)
    *bytes_read = p - str;
669 670 671 672
  else
    {
      if ((p - str) != len) 
	{
Havoc Pennington's avatar
Havoc Pennington committed
673 674
          if (!have_error)
            {
675
	      if (error)
676 677
		g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
                                     _("Partial character sequence at end of input"));
Havoc Pennington's avatar
Havoc Pennington committed
678 679
              have_error = TRUE;
            }
680 681
	}
    }
682 683 684 685 686 687 688 689 690 691 692 693 694

  if (bytes_written)
    *bytes_written = outp - dest;	/* Doesn't include '\0' */

  if (have_error)
    {
      g_free (dest);
      return NULL;
    }
  else
    return dest;
}

695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
/**
 * g_convert:
 * @str:           the string to convert
 * @len:           the length of the string, or -1 if the string is 
 *                 nul-terminated<footnote id="nul-unsafe">
                     <para>
                       Note that some encodings may allow nul bytes to 
                       occur inside strings. In that case, using -1 for 
                       the @len parameter is unsafe.
                     </para>
                   </footnote>. 
 * @to_codeset:    name of character set into which to convert @str
 * @from_codeset:  character set of @str.
 * @bytes_read:    location to store the number of bytes in the
 *                 input string that were successfully converted, or %NULL.
 *                 Even if the conversion was successful, this may be 
 *                 less than @len if there were partial characters
 *                 at the end of the input. If the error
 *                 #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
 *                 stored will the byte offset after the last valid
 *                 input sequence.
 * @bytes_written: the number of bytes stored in the output buffer (not 
 *                 including the terminating nul).
 * @error:         location to store the error occuring, or %NULL to ignore
 *                 errors. Any of the errors in #GConvertError may occur.
 *
 * Converts a string from one character set to another.
 *
 * Note that you should use g_iconv() for streaming 
 * conversions<footnoteref linkend="streaming-state"/>.
 *
 * Return value: If the conversion was successful, a newly allocated
727 728
 *               nul-terminated string, which must be freed with
 *               g_free(). Otherwise %NULL and @error will be set.
729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
 **/
gchar*
g_convert (const gchar *str,
           gssize       len,  
           const gchar *to_codeset,
           const gchar *from_codeset,
           gsize       *bytes_read, 
	   gsize       *bytes_written, 
	   GError     **error)
{
  gchar *res;
  GIConv cd;

  g_return_val_if_fail (str != NULL, NULL);
  g_return_val_if_fail (to_codeset != NULL, NULL);
  g_return_val_if_fail (from_codeset != NULL, NULL);
  
  cd = open_converter (to_codeset, from_codeset, error);

  if (cd == (GIConv) -1)
    {
      if (bytes_read)
        *bytes_read = 0;
      
      if (bytes_written)
        *bytes_written = 0;
      
      return NULL;
    }

  res = g_convert_with_iconv (str, len, cd,
			      bytes_read, bytes_written,
			      error);

  close_converter (cd);

  return res;
}

768 769 770
/**
 * g_convert_with_fallback:
 * @str:          the string to convert
771 772
 * @len:          the length of the string, or -1 if the string is 
 *                nul-terminated<footnoteref linkend="nul-unsafe"/>. 
773 774 775
 * @to_codeset:   name of character set into which to convert @str
 * @from_codeset: character set of @str.
 * @fallback:     UTF-8 string to use in place of character not
776 777 778 779
 *                present in the target encoding. (The string must be
 *                representable in the target encoding). 
                  If %NULL, characters not in the target encoding will 
                  be represented as Unicode escapes \uxxxx or \Uxxxxyyyy.
780 781
 * @bytes_read:   location to store the number of bytes in the
 *                input string that were successfully converted, or %NULL.
Matthias Clasen's avatar
Matthias Clasen committed
782
 *                Even if the conversion was successful, this may be 
783
 *                less than @len if there were partial characters
784
 *                at the end of the input.
785
 * @bytes_written: the number of bytes stored in the output buffer (not 
Matthias Clasen's avatar
Matthias Clasen committed
786
 *                including the terminating nul).
787 788 789
 * @error:        location to store the error occuring, or %NULL to ignore
 *                errors. Any of the errors in #GConvertError may occur.
 *
Matthias Clasen's avatar
Matthias Clasen committed
790
 * Converts a string from one character set to another, possibly
791 792 793
 * including fallback sequences for characters not representable
 * in the output. Note that it is not guaranteed that the specification
 * for the fallback sequences in @fallback will be honored. Some
794
 * systems may do an approximate conversion from @from_codeset
795
 * to @to_codeset in their iconv() functions, 
Owen Taylor's avatar
Owen Taylor committed
796
 * in which case GLib will simply return that approximate conversion.
797
 *
798 799 800
 * Note that you should use g_iconv() for streaming 
 * conversions<footnoteref linkend="streaming-state"/>.
 *
801
 * Return value: If the conversion was successful, a newly allocated
802 803
 *               nul-terminated string, which must be freed with
 *               g_free(). Otherwise %NULL and @error will be set.
804 805 806
 **/
gchar*
g_convert_with_fallback (const gchar *str,
807
			 gssize       len,    
808 809
			 const gchar *to_codeset,
			 const gchar *from_codeset,
810
			 const gchar *fallback,
811 812
			 gsize       *bytes_read,
			 gsize       *bytes_written,
813 814 815 816 817 818 819
			 GError     **error)
{
  gchar *utf8;
  gchar *dest;
  gchar *outp;
  const gchar *insert_str = NULL;
  const gchar *p;
820
  gsize inbytes_remaining;   
821
  const gchar *save_p = NULL;
822 823 824
  gsize save_inbytes = 0;
  gsize outbytes_remaining; 
  gsize err;
825
  GIConv cd;
826
  gsize outbuf_size;
827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854
  gboolean have_error = FALSE;
  gboolean done = FALSE;

  GError *local_error = NULL;
  
  g_return_val_if_fail (str != NULL, NULL);
  g_return_val_if_fail (to_codeset != NULL, NULL);
  g_return_val_if_fail (from_codeset != NULL, NULL);
     
  if (len < 0)
    len = strlen (str);
  
  /* Try an exact conversion; we only proceed if this fails
   * due to an illegal sequence in the input string.
   */
  dest = g_convert (str, len, to_codeset, from_codeset, 
		    bytes_read, bytes_written, &local_error);
  if (!local_error)
    return dest;

  if (!g_error_matches (local_error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE))
    {
      g_propagate_error (error, local_error);
      return NULL;
    }
  else
    g_error_free (local_error);

Havoc Pennington's avatar
Havoc Pennington committed
855 856
  local_error = NULL;
  
857 858 859 860
  /* No go; to proceed, we need a converter from "UTF-8" to
   * to_codeset, and the string as UTF-8.
   */
  cd = open_converter (to_codeset, "UTF-8", error);
861
  if (cd == (GIConv) -1)
862 863 864 865 866 867 868 869 870 871 872 873 874
    {
      if (bytes_read)
        *bytes_read = 0;
      
      if (bytes_written)
        *bytes_written = 0;
      
      return NULL;
    }

  utf8 = g_convert (str, len, "UTF-8", from_codeset, 
		    bytes_read, &inbytes_remaining, error);
  if (!utf8)
875
    {
876
      close_converter (cd);
877 878 879 880
      if (bytes_written)
        *bytes_written = 0;
      return NULL;
    }
881 882 883 884 885 886 887 888 889 890

  /* Now the heart of the code. We loop through the UTF-8 string, and
   * whenever we hit an offending character, we form fallback, convert
   * the fallback to the target codeset, and then go back to
   * converting the original string after finishing with the fallback.
   *
   * The variables save_p and save_inbytes store the input state
   * for the original string while we are converting the fallback
   */
  p = utf8;
891

892 893
  outbuf_size = len + NUL_TERMINATOR_LENGTH;
  outbytes_remaining = outbuf_size - NUL_TERMINATOR_LENGTH;
894 895 896 897
  outp = dest = g_malloc (outbuf_size);

  while (!done && !have_error)
    {
898
      gsize inbytes_tmp = inbytes_remaining;
899
      err = g_iconv (cd, (char **)&p, &inbytes_tmp, &outp, &outbytes_remaining);
900
      inbytes_remaining = inbytes_tmp;
901

902
      if (err == (gsize) -1)
903 904 905 906 907 908 909 910
	{
	  switch (errno)
	    {
	    case EINVAL:
	      g_assert_not_reached();
	      break;
	    case E2BIG:
	      {
911
		gsize used = outp - dest;
912

913 914 915 916
		outbuf_size *= 2;
		dest = g_realloc (dest, outbuf_size);
		
		outp = dest + used;
917
		outbytes_remaining = outbuf_size - used - NUL_TERMINATOR_LENGTH;
918 919 920 921 922 923 924 925 926
		
		break;
	      }
	    case EILSEQ:
	      if (save_p)
		{
		  /* Error converting fallback string - fatal
		   */
		  g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
927
			       _("Cannot convert fallback '%s' to codeset '%s'"),
928 929 930 931
			       insert_str, to_codeset);
		  have_error = TRUE;
		  break;
		}
932
	      else if (p)
933 934 935 936
		{
		  if (!fallback)
		    { 
		      gunichar ch = g_utf8_get_char (p);
937
		      insert_str = g_strdup_printf (ch < 0x10000 ? "\\u%04x" : "\\U%08x",
938 939 940 941 942 943 944 945 946
						    ch);
		    }
		  else
		    insert_str = fallback;
		  
		  save_p = g_utf8_next_char (p);
		  save_inbytes = inbytes_remaining - (save_p - p);
		  p = insert_str;
		  inbytes_remaining = strlen (p);
947
		  break;
948
		}
949
	      /* fall thru if p is NULL */
950
	    default:
Christian Persch's avatar
Christian Persch committed
951 952 953 954 955 956 957 958
              {
                int errsv = errno;

                g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
                             _("Error during conversion: %s"),
                             g_strerror (errsv));
              }

959 960 961 962 963 964 965 966 967 968 969 970 971 972
	      have_error = TRUE;
	      break;
	    }
	}
      else
	{
	  if (save_p)
	    {
	      if (!fallback)
		g_free ((gchar *)insert_str);
	      p = save_p;
	      inbytes_remaining = save_inbytes;
	      save_p = NULL;
	    }
973 974 975 976 977 978
	  else if (p)
	    {
	      /* call g_iconv with NULL inbuf to cleanup shift state */
	      p = NULL;
	      inbytes_remaining = 0;
	    }
979 980 981 982 983 984 985
	  else
	    done = TRUE;
	}
    }

  /* Cleanup
   */
986
  memset (outp, 0, NUL_TERMINATOR_LENGTH);
987
  
988
  close_converter (cd);
989 990

  if (bytes_written)
991
    *bytes_written = outp - dest;	/* Doesn't include '\0' */
992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004

  g_free (utf8);

  if (have_error)
    {
      if (save_p && !fallback)
	g_free ((gchar *)insert_str);
      g_free (dest);
      return NULL;
    }
  else
    return dest;
}
1005 1006 1007 1008

/*
 * g_locale_to_utf8
 *
1009 1010 1011
 * 
 */

1012 1013 1014 1015
static gchar *
strdup_len (const gchar *string,
	    gssize       len,
	    gsize       *bytes_written,
1016 1017
	    gsize       *bytes_read,
	    GError      **error)
1018 1019 1020 1021
	 
{
  gsize real_len;

1022
  if (!g_utf8_validate (string, len, NULL))
1023 1024 1025 1026 1027 1028
    {
      if (bytes_read)
	*bytes_read = 0;
      if (bytes_written)
	*bytes_written = 0;

1029 1030
      g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
                           _("Invalid byte sequence in conversion input"));
1031 1032 1033
      return NULL;
    }
  
1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
  if (len < 0)
    real_len = strlen (string);
  else
    {
      real_len = 0;
      
      while (real_len < len && string[real_len])
	real_len++;
    }
  
  if (bytes_read)
    *bytes_read = real_len;
  if (bytes_written)
    *bytes_written = real_len;

  return g_strndup (string, real_len);
}

1052 1053
/**
 * g_locale_to_utf8:
Tor Lillqvist's avatar
Tor Lillqvist committed
1054 1055
 * @opsysstring:   a string in the encoding of the current locale. On Windows
 *                 this means the system codepage.
1056
 * @len:           the length of the string, or -1 if the string is
1057
 *                 nul-terminated<footnoteref linkend="nul-unsafe"/>. 
1058 1059
 * @bytes_read:    location to store the number of bytes in the
 *                 input string that were successfully converted, or %NULL.
Matthias Clasen's avatar
Matthias Clasen committed
1060
 *                 Even if the conversion was successful, this may be 
1061
 *                 less than @len if there were partial characters
1062
 *                 at the end of the input. If the error
1063 1064
 *                 #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
 *                 stored will the byte offset after the last valid
1065
 *                 input sequence.
1066
 * @bytes_written: the number of bytes stored in the output buffer (not 
Matthias Clasen's avatar
Matthias Clasen committed
1067
 *                 including the terminating nul).
1068
 * @error:         location to store the error occuring, or %NULL to ignore
1069 1070
 *                 errors. Any of the errors in #GConvertError may occur.
 * 
1071 1072
 * Converts a string which is in the encoding used for strings by
 * the C runtime (usually the same as that used by the operating
1073 1074
 * system) in the <link linkend="setlocale">current locale</link> into a
 * UTF-8 string.
1075 1076 1077
 * 
 * Return value: The converted string, or %NULL on an error.
 **/
1078
gchar *
1079
g_locale_to_utf8 (const gchar  *opsysstring,
1080 1081 1082
		  gssize        len,            
		  gsize        *bytes_read,    
		  gsize        *bytes_written,
1083
		  GError      **error)
1084
{
1085
  const char *charset;
1086 1087

  if (g_get_charset (&charset))
1088
    return strdup_len (opsysstring, len, bytes_read, bytes_written, error);
1089 1090 1091
  else
    return g_convert (opsysstring, len, 
		      "UTF-8", charset, bytes_read, bytes_written, error);
1092 1093
}

1094 1095 1096 1097
/**
 * g_locale_from_utf8:
 * @utf8string:    a UTF-8 encoded string 
 * @len:           the length of the string, or -1 if the string is
1098
 *                 nul-terminated<footnoteref linkend="nul-unsafe"/>. 
1099 1100
 * @bytes_read:    location to store the number of bytes in the
 *                 input string that were successfully converted, or %NULL.
Matthias Clasen's avatar
Matthias Clasen committed
1101
 *                 Even if the conversion was successful, this may be 
1102
 *                 less than @len if there were partial characters
1103
 *                 at the end of the input. If the error
1104 1105
 *                 #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
 *                 stored will the byte offset after the last valid
1106
 *                 input sequence.
1107
 * @bytes_written: the number of bytes stored in the output buffer (not 
Matthias Clasen's avatar
Matthias Clasen committed
1108
 *                 including the terminating nul).
1109
 * @error:         location to store the error occuring, or %NULL to ignore
1110 1111 1112 1113
 *                 errors. Any of the errors in #GConvertError may occur.
 * 
 * Converts a string from UTF-8 to the encoding used for strings by
 * the C runtime (usually the same as that used by the operating
1114 1115
 * system) in the <link linkend="setlocale">current locale</link>. On
 * Windows this means the system codepage.
1116 1117 1118
 * 
 * Return value: The converted string, or %NULL on an error.
 **/
1119
gchar *
1120
g_locale_from_utf8 (const gchar *utf8string,
1121 1122 1123
		    gssize       len,            
		    gsize       *bytes_read,    
		    gsize       *bytes_written,
1124
		    GError     **error)
1125
{
1126
  const gchar *charset;
1127 1128

  if (g_get_charset (&charset))
1129
    return strdup_len (utf8string, len, bytes_read, bytes_written, error);
1130 1131 1132
  else
    return g_convert (utf8string, len,
		      charset, "UTF-8", bytes_read, bytes_written, error);
1133 1134
}

1135
#ifndef G_PLATFORM_WIN32
1136

1137 1138 1139 1140 1141
typedef struct _GFilenameCharsetCache GFilenameCharsetCache;

struct _GFilenameCharsetCache {
  gboolean is_utf8;
  gchar *charset;
1142
  gchar **filename_charsets;
1143 1144 1145 1146 1147 1148 1149
};

static void
filename_charset_cache_free (gpointer data)
{
  GFilenameCharsetCache *cache = data;
  g_free (cache->charset);
1150
  g_strfreev (cache->filename_charsets);
1151 1152 1153
  g_free (cache);
}

Matthias Clasen's avatar
Matthias Clasen committed
1154
/**
1155 1156
 * g_get_filename_charsets:
 * @charsets: return location for the %NULL-terminated list of encoding names
1157
 *
1158 1159 1160
 * Determines the preferred character sets used for filenames.
 * The first character set from the @charsets is the filename encoding, the
 * subsequent character sets are used when trying to generate a displayable
Tor Lillqvist's avatar
Tor Lillqvist committed
1161
 * representation of a filename, see g_filename_display_name().
1162
 *
1163 1164 1165 1166 1167
 * On Unix, the character sets are determined by consulting the
 * environment variables <envar>G_FILENAME_ENCODING</envar> and
 * <envar>G_BROKEN_FILENAMES</envar>. On Windows, the character set
 * used in the GLib API is always UTF-8 and said environment variables
 * have no effect.
1168
 *
1169 1170 1171 1172 1173 1174 1175
 * <envar>G_FILENAME_ENCODING</envar> may be set to a comma-separated list 
 * of character set names. The special token "&commat;locale" is taken to 
 * mean the character set for the <link linkend="setlocale">current 
 * locale</link>. If <envar>G_FILENAME_ENCODING</envar> is not set, but 
 * <envar>G_BROKEN_FILENAMES</envar> is, the character set of the current 
 * locale is taken as the filename encoding. If neither environment variable 
 * is set, UTF-8 is taken as the filename encoding, but the character
1176 1177 1178
 * set of the current locale is also put in the list of encodings.
 *
 * The returned @charsets belong to GLib and must not be freed.
1179 1180
 *
 * Note that on Unix, regardless of the locale character set or
1181 1182
 * <envar>G_FILENAME_ENCODING</envar> value, the actual file names present 
 * on a system might be in any random encoding or just gibberish.
1183
 *
1184 1185 1186
 * Return value: %TRUE if the filename encoding is UTF-8.
 * 
 * Since: 2.6
1187
 */
1188 1189
gboolean
g_get_filename_charsets (G_CONST_RETURN gchar ***filename_charsets)
1190
{
1191 1192 1193
  static GStaticPrivate cache_private = G_STATIC_PRIVATE_INIT;
  GFilenameCharsetCache *cache = g_static_private_get (&cache_private);
  const gchar *charset;
1194

1195
  if (!cache)
1196
    {
1197 1198 1199 1200 1201 1202 1203 1204 1205
      cache = g_new0 (GFilenameCharsetCache, 1);
      g_static_private_set (&cache_private, cache, filename_charset_cache_free);
    }

  g_get_charset (&charset);

  if (!(cache->charset && strcmp (cache->charset, charset) == 0))
    {
      const gchar *new_charset;
1206 1207
      gchar *p;
      gint i;
1208

1209
      g_free (cache->charset);
1210
      g_strfreev (cache->filename_charsets);
1211
      cache->charset = g_strdup (charset);
1212 1213
      
      p = getenv ("G_FILENAME_ENCODING");
1214
      if (p != NULL && p[0] != '\0') 
1215
	{
1216 1217
	  cache->filename_charsets = g_strsplit (p, ",", 0);
	  cache->is_utf8 = (strcmp (cache->filename_charsets[0], "UTF-8") == 0);
1218

1219
	  for (i = 0; cache->filename_charsets[i]; i++)
1220
	    {
1221 1222 1223 1224 1225 1226
	      if (strcmp ("@locale", cache->filename_charsets[i]) == 0)
		{
		  g_get_charset (&new_charset);
		  g_free (cache->filename_charsets[i]);
		  cache->filename_charsets[i] = g_strdup (new_charset);
		}
1227 1228 1229
	    }
	}
      else if (getenv ("G_BROKEN_FILENAMES") != NULL)
1230
	{
1231
	  cache->filename_charsets = g_new0 (gchar *, 2);
1232
	  cache->is_utf8 = g_get_charset (&new_charset);
1233
	  cache->filename_charsets[0] = g_strdup (new_charset);
1234
	}
1235 1236
      else 
	{
1237
	  cache->filename_charsets = g_new0 (gchar *, 3);
1238
	  cache->is_utf8 = TRUE;
1239 1240 1241
	  cache->filename_charsets[0] = g_strdup ("UTF-8");
	  if (!g_get_charset (&new_charset))
	    cache->filename_charsets[1] = g_strdup (new_charset);
1242 1243
	}
    }
1244

1245 1246
  if (filename_charsets)
    *filename_charsets = (const gchar **)cache->filename_charsets;
1247

1248
  return cache->is_utf8;
1249
}
1250

1251
#else /* G_PLATFORM_WIN32 */
1252

1253 1254
gboolean
g_get_filename_charsets (G_CONST_RETURN gchar ***filename_charsets) 
1255
{
1256
  static const gchar *charsets[] = {
1257 1258 1259 1260
    "UTF-8",
    NULL
  };

1261 1262
#ifdef G_OS_WIN32
  /* On Windows GLib pretends that the filename charset is UTF-8 */
1263 1264 1265
  if (filename_charsets)
    *filename_charsets = charsets;

1266 1267
  return TRUE;
#else
1268 1269
  gboolean result;

1270
  /* Cygwin works like before */
1271 1272 1273 1274 1275 1276
  result = g_get_charset (&(charsets[0]));

  if (filename_charsets)
    *filename_charsets = charsets;

  return result;
1277 1278 1279
#endif
}

1280
#endif /* G_PLATFORM_WIN32 */
1281 1282

static gboolean
1283
get_filename_charset (const gchar **filename_charset)
1284
{
1285 1286 1287 1288
  const gchar **charsets;
  gboolean is_utf8;
  
  is_utf8 = g_get_filename_charsets (&charsets);
1289

1290 1291 1292 1293 1294
  if (filename_charset)
    *filename_charset = charsets[0];
  
  return is_utf8;
}
1295 1296 1297 1298 1299

/* This is called from g_thread_init(). It's used to
 * initialize some static data in a threadsafe way.
 */
void 
1300
_g_convert_thread_init (void)
1301
{
1302
  const gchar **dummy;
Matthias Clasen's avatar
Matthias Clasen committed
1303
  (void) g_get_filename_charsets (&dummy);
1304 1305
}

1306 1307 1308 1309
/**
 * g_filename_to_utf8:
 * @opsysstring:   a string in the encoding for filenames
 * @len:           the length of the string, or -1 if the string is
1310
 *                 nul-terminated<footnoteref linkend="nul-unsafe"/>. 
1311 1312
 * @bytes_read:    location to store the number of bytes in the
 *                 input string that were successfully converted, or %NULL.
Matthias Clasen's avatar
Matthias Clasen committed
1313
 *                 Even if the conversion was successful, this may be 
1314
 *                 less than @len if there were partial characters
1315
 *                 at the end of the input. If the error
1316 1317
 *                 #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
 *                 stored will the byte offset after the last valid
1318
 *                 input sequence.
1319
 * @bytes_written: the number of bytes stored in the output buffer (not 
Matthias Clasen's avatar
Matthias Clasen committed
1320
 *                 including the terminating nul).
1321
 * @error:         location to store the error occuring, or %NULL to ignore
1322 1323
 *                 errors. Any of the errors in #GConvertError may occur.
 * 
Tor Lillqvist's avatar