gconvert.c 56.2 KB
Newer Older
1 2 3 4
/* GLIB - Library of useful routines for C programming
 *
 * gconvert.c: Convert between character sets using iconv
 * Copyright Red Hat Inc., 2000
5
 * Authors: Havoc Pennington <hp@redhat.com>, Owen Taylor <otaylor@redhat.com>
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 */

23
#include "config.h"
Owen Taylor's avatar
1.3.14  
Owen Taylor committed
24

25 26 27
#include "glib.h"

#ifndef G_OS_WIN32
28
#include <iconv.h>
29
#endif
30
#include <errno.h>
31
#include <stdio.h>
32
#include <string.h>
33 34
#include <stdlib.h>

35
#include "gprintfint.h"
36
#include "gthreadprivate.h"
37
#include "gunicode.h"
Tor Lillqvist's avatar
Tor Lillqvist committed
38

39 40 41 42
#ifdef G_OS_WIN32
#include "win_iconv.c"
#endif

43 44
#ifdef G_PLATFORM_WIN32
#define STRICT
45
#include <windows.h>
46
#undef STRICT
47
#endif
48

Owen Taylor's avatar
Owen Taylor committed
49
#include "glibintl.h"
50

51 52
#if defined(USE_LIBICONV_GNU) && !defined (_LIBICONV_H)
#error GNU libiconv in use but included iconv.h not from libiconv
53
#endif
54 55
#if !defined(USE_LIBICONV_GNU) && defined (_LIBICONV_H)
#error GNU libiconv not in use but included iconv.h is from libiconv
56 57
#endif

58 59
#include "galias.h"

60 61 62 63 64 65
/* We try to terminate strings in unknown charsets with this many zero bytes
 * to ensure that multibyte strings really are nul-terminated when we return
 * them from g_convert() and friends.
 */
#define NUL_TERMINATOR_LENGTH 4

66
GQuark 
67
g_convert_error_quark (void)
68
{
69
  return g_quark_from_static_string ("g_convert_error");
70 71
}

72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
static gboolean
try_conversion (const char *to_codeset,
		const char *from_codeset,
		iconv_t    *cd)
{
  *cd = iconv_open (to_codeset, from_codeset);

  if (*cd == (iconv_t)-1 && errno == EINVAL)
    return FALSE;
  else
    return TRUE;
}

static gboolean
try_to_aliases (const char **to_aliases,
		const char  *from_codeset,
		iconv_t     *cd)
{
  if (to_aliases)
    {
      const char **p = to_aliases;
      while (*p)
	{
	  if (try_conversion (*p, from_codeset, cd))
	    return TRUE;

	  p++;
	}
    }

  return FALSE;
}

105 106
G_GNUC_INTERNAL extern const char ** 
_g_charset_get_aliases (const char *canonical_name);
107

Havoc Pennington's avatar
docs  
Havoc Pennington committed
108 109 110 111 112
/**
 * g_iconv_open:
 * @to_codeset: destination codeset
 * @from_codeset: source codeset
 * 
113
 * Same as the standard UNIX routine iconv_open(), but
114
 * may be implemented via libiconv on UNIX flavors that lack
Havoc Pennington's avatar
docs  
Havoc Pennington committed
115 116
 * a native implementation.
 * 
Owen Taylor's avatar
Owen Taylor committed
117
 * GLib provides g_convert() and g_locale_to_utf8() which are likely
Havoc Pennington's avatar
docs  
Havoc Pennington committed
118 119
 * more convenient than the raw iconv wrappers.
 * 
120 121
 * Return value: a "conversion descriptor", or (GIConv)-1 if
 *  opening the converter failed.
Havoc Pennington's avatar
docs  
Havoc Pennington committed
122
 **/
123 124 125 126
GIConv
g_iconv_open (const gchar  *to_codeset,
	      const gchar  *from_codeset)
{
127
  iconv_t cd;
128
  
129 130 131
  if (!try_conversion (to_codeset, from_codeset, &cd))
    {
      const char **to_aliases = _g_charset_get_aliases (to_codeset);
132
      const char **from_aliases = _g_charset_get_aliases (from_codeset);
133 134 135 136 137 138 139

      if (from_aliases)
	{
	  const char **p = from_aliases;
	  while (*p)
	    {
	      if (try_conversion (to_codeset, *p, &cd))
140
		goto out;
141 142

	      if (try_to_aliases (to_aliases, *p, &cd))
143
		goto out;
144 145 146 147 148 149

	      p++;
	    }
	}

      if (try_to_aliases (to_aliases, from_codeset, &cd))
150
	goto out;
151 152
    }

153
 out:
154
  return (cd == (iconv_t)-1) ? (GIConv)-1 : (GIConv)cd;
155 156
}

Havoc Pennington's avatar
docs  
Havoc Pennington committed
157 158 159 160 161 162 163 164
/**
 * g_iconv:
 * @converter: conversion descriptor from g_iconv_open()
 * @inbuf: bytes to convert
 * @inbytes_left: inout parameter, bytes remaining to convert in @inbuf
 * @outbuf: converted output bytes
 * @outbytes_left: inout parameter, bytes available to fill in @outbuf
 * 
165
 * Same as the standard UNIX routine iconv(), but
166
 * may be implemented via libiconv on UNIX flavors that lack
Havoc Pennington's avatar
docs  
Havoc Pennington committed
167 168
 * a native implementation.
 *
Owen Taylor's avatar
Owen Taylor committed
169
 * GLib provides g_convert() and g_locale_to_utf8() which are likely
Havoc Pennington's avatar
docs  
Havoc Pennington committed
170 171 172 173
 * more convenient than the raw iconv wrappers.
 * 
 * Return value: count of non-reversible conversions, or -1 on error
 **/
174
gsize 
175 176
g_iconv (GIConv   converter,
	 gchar  **inbuf,
177
	 gsize   *inbytes_left,
178
	 gchar  **outbuf,
179
	 gsize   *outbytes_left)
180 181 182 183 184 185
{
  iconv_t cd = (iconv_t)converter;

  return iconv (cd, inbuf, inbytes_left, outbuf, outbytes_left);
}

Havoc Pennington's avatar
docs  
Havoc Pennington committed
186 187 188 189
/**
 * g_iconv_close:
 * @converter: a conversion descriptor from g_iconv_open()
 *
190
 * Same as the standard UNIX routine iconv_close(), but
191
 * may be implemented via libiconv on UNIX flavors that lack
Havoc Pennington's avatar
docs  
Havoc Pennington committed
192
 * a native implementation. Should be called to clean up
Matthias Clasen's avatar
Matthias Clasen committed
193
 * the conversion descriptor from g_iconv_open() when
Havoc Pennington's avatar
docs  
Havoc Pennington committed
194 195
 * you are done converting things.
 *
Owen Taylor's avatar
Owen Taylor committed
196
 * GLib provides g_convert() and g_locale_to_utf8() which are likely
Havoc Pennington's avatar
docs  
Havoc Pennington committed
197 198 199 200
 * more convenient than the raw iconv wrappers.
 * 
 * Return value: -1 on error, 0 on success
 **/
201 202 203 204 205 206 207 208
gint
g_iconv_close (GIConv converter)
{
  iconv_t cd = (iconv_t)converter;

  return iconv_close (cd);
}

209

Matthias Clasen's avatar
Matthias Clasen committed
210 211
#ifdef NEED_ICONV_CACHE

212 213 214 215 216 217
#define ICONV_CACHE_SIZE   (16)

struct _iconv_cache_bucket {
  gchar *key;
  guint32 refcount;
  gboolean used;
218
  GIConv cd;
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
};

static GList *iconv_cache_list;
static GHashTable *iconv_cache;
static GHashTable *iconv_open_hash;
static guint iconv_cache_size = 0;
G_LOCK_DEFINE_STATIC (iconv_cache_lock);

/* caller *must* hold the iconv_cache_lock */
static void
iconv_cache_init (void)
{
  static gboolean initialized = FALSE;
  
  if (initialized)
    return;
  
  iconv_cache_list = NULL;
  iconv_cache = g_hash_table_new (g_str_hash, g_str_equal);
  iconv_open_hash = g_hash_table_new (g_direct_hash, g_direct_equal);
  
  initialized = TRUE;
}


244
/*
245 246 247 248 249 250 251
 * iconv_cache_bucket_new:
 * @key: cache key
 * @cd: iconv descriptor
 *
 * Creates a new cache bucket, inserts it into the cache and
 * increments the cache size.
 *
252 253
 * This assumes ownership of @key.
 *
254 255
 * Returns a pointer to the newly allocated cache bucket.
 **/
256
static struct _iconv_cache_bucket *
257
iconv_cache_bucket_new (gchar *key, GIConv cd)
258 259 260 261
{
  struct _iconv_cache_bucket *bucket;
  
  bucket = g_new (struct _iconv_cache_bucket, 1);
262
  bucket->key = key;
263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
  bucket->refcount = 1;
  bucket->used = TRUE;
  bucket->cd = cd;
  
  g_hash_table_insert (iconv_cache, bucket->key, bucket);
  
  /* FIXME: if we sorted the list so items with few refcounts were
     first, then we could expire them faster in iconv_cache_expire_unused () */
  iconv_cache_list = g_list_prepend (iconv_cache_list, bucket);
  
  iconv_cache_size++;
  
  return bucket;
}


279
/*
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
 * iconv_cache_bucket_expire:
 * @node: cache bucket's node
 * @bucket: cache bucket
 *
 * Expires a single cache bucket @bucket. This should only ever be
 * called on a bucket that currently has no used iconv descriptors
 * open.
 *
 * @node is not a required argument. If @node is not supplied, we
 * search for it ourselves.
 **/
static void
iconv_cache_bucket_expire (GList *node, struct _iconv_cache_bucket *bucket)
{
  g_hash_table_remove (iconv_cache, bucket->key);
  
  if (node == NULL)
    node = g_list_find (iconv_cache_list, bucket);
  
  g_assert (node != NULL);
  
  if (node->prev)
    {
      node->prev->next = node->next;
      if (node->next)
        node->next->prev = node->prev;
    }
  else
    {
      iconv_cache_list = node->next;
      if (node->next)
        node->next->prev = NULL;
    }
  
  g_list_free_1 (node);
  
  g_free (bucket->key);
  g_iconv_close (bucket->cd);
  g_free (bucket);
  
  iconv_cache_size--;
}


324
/*
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
 * iconv_cache_expire_unused:
 *
 * Expires as many unused cache buckets as it needs to in order to get
 * the total number of buckets < ICONV_CACHE_SIZE.
 **/
static void
iconv_cache_expire_unused (void)
{
  struct _iconv_cache_bucket *bucket;
  GList *node, *next;
  
  node = iconv_cache_list;
  while (node && iconv_cache_size >= ICONV_CACHE_SIZE)
    {
      next = node->next;
      
      bucket = node->data;
      if (bucket->refcount == 0)
        iconv_cache_bucket_expire (node, bucket);
      
      node = next;
    }
}

Havoc Pennington's avatar
Havoc Pennington committed
349
static GIConv
350
open_converter (const gchar *to_codeset,
351
		const gchar *from_codeset,
352 353
		GError     **error)
{
354
  struct _iconv_cache_bucket *bucket;
355
  gchar *key, *dyn_key, auto_key[80];
356
  GIConv cd;
357
  gsize len_from_codeset, len_to_codeset;
358 359
  
  /* create our key */
360 361 362 363 364 365 366 367 368 369 370 371 372
  len_from_codeset = strlen (from_codeset);
  len_to_codeset = strlen (to_codeset);
  if (len_from_codeset + len_to_codeset + 2 < sizeof (auto_key))
    {
      key = auto_key;
      dyn_key = NULL;
    }
  else
    key = dyn_key = g_malloc (len_from_codeset + len_to_codeset + 2);
  memcpy (key, from_codeset, len_from_codeset);
  key[len_from_codeset] = ':';
  strcpy (key + len_from_codeset + 1, to_codeset);

373 374 375 376 377 378 379
  G_LOCK (iconv_cache_lock);
  
  /* make sure the cache has been initialized */
  iconv_cache_init ();
  
  bucket = g_hash_table_lookup (iconv_cache, key);
  if (bucket)
380
    {
381 382
      g_free (dyn_key);

383 384 385
      if (bucket->used)
        {
          cd = g_iconv_open (to_codeset, from_codeset);
386
          if (cd == (GIConv) -1)
387 388
            goto error;
        }
389
      else
390
        {
391 392 393 394
	  /* Apparently iconv on Solaris <= 7 segfaults if you pass in
	   * NULL for anything but inbuf; work around that. (NULL outbuf
	   * or NULL *outbuf is allowed by Unix98.)
	   */
395
	  gsize inbytes_left = 0;
396
	  gchar *outbuf = NULL;
397
	  gsize outbytes_left = 0;
398
		
399 400 401 402
          cd = bucket->cd;
          bucket->used = TRUE;
          
          /* reset the descriptor */
403
          g_iconv (cd, NULL, &inbytes_left, &outbuf, &outbytes_left);
404 405 406
        }
      
      bucket->refcount++;
407
    }
408 409 410
  else
    {
      cd = g_iconv_open (to_codeset, from_codeset);
411 412 413 414 415
      if (cd == (GIConv) -1) 
	{
	  g_free (dyn_key);
	  goto error;
	}
416 417
      
      iconv_cache_expire_unused ();
418 419

      bucket = iconv_cache_bucket_new (dyn_key ? dyn_key : g_strdup (key), cd);
420 421 422 423 424 425 426 427 428 429 430 431 432
    }
  
  g_hash_table_insert (iconv_open_hash, cd, bucket->key);
  
  G_UNLOCK (iconv_cache_lock);
  
  return cd;
  
 error:
  
  G_UNLOCK (iconv_cache_lock);
  
  /* Something went wrong.  */
433 434 435 436 437 438 439 440 441 442 443
  if (error)
    {
      if (errno == EINVAL)
	g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_NO_CONVERSION,
		     _("Conversion from character set '%s' to '%s' is not supported"),
		     from_codeset, to_codeset);
      else
	g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
		     _("Could not open converter from '%s' to '%s'"),
		     from_codeset, to_codeset);
    }
444
  
445
  return cd;
446
}
447

448 449 450 451 452
static int
close_converter (GIConv converter)
{
  struct _iconv_cache_bucket *bucket;
  const gchar *key;
453
  GIConv cd;
454
  
455
  cd = converter;
456
  
457
  if (cd == (GIConv) -1)
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494
    return 0;
  
  G_LOCK (iconv_cache_lock);
  
  key = g_hash_table_lookup (iconv_open_hash, cd);
  if (key)
    {
      g_hash_table_remove (iconv_open_hash, cd);
      
      bucket = g_hash_table_lookup (iconv_cache, key);
      g_assert (bucket);
      
      bucket->refcount--;
      
      if (cd == bucket->cd)
        bucket->used = FALSE;
      else
        g_iconv_close (cd);
      
      if (!bucket->refcount && iconv_cache_size > ICONV_CACHE_SIZE)
        {
          /* expire this cache bucket */
          iconv_cache_bucket_expire (NULL, bucket);
        }
    }
  else
    {
      G_UNLOCK (iconv_cache_lock);
      
      g_warning ("This iconv context wasn't opened using open_converter");
      
      return g_iconv_close (converter);
    }
  
  G_UNLOCK (iconv_cache_lock);
  
  return 0;
495 496
}

Matthias Clasen's avatar
Matthias Clasen committed
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537
#else  /* !NEED_ICONV_CACHE */

static GIConv
open_converter (const gchar *to_codeset,
		const gchar *from_codeset,
		GError     **error)
{
  GIConv cd;

  cd = g_iconv_open (to_codeset, from_codeset);

  if (cd == (GIConv) -1)
    {
      /* Something went wrong.  */
      if (error)
	{
	  if (errno == EINVAL)
	    g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_NO_CONVERSION,
			 _("Conversion from character set '%s' to '%s' is not supported"),
			 from_codeset, to_codeset);
	  else
	    g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
			 _("Could not open converter from '%s' to '%s'"),
			 from_codeset, to_codeset);
	}
    }
  
  return cd;
}

static int
close_converter (GIConv cd)
{
  if (cd == (GIConv) -1)
    return 0;
  
  return g_iconv_close (cd);  
}

#endif /* NEED_ICONV_CACHE */

538 539 540
/**
 * g_convert_with_iconv:
 * @str:           the string to convert
541 542
 * @len:           the length of the string, or -1 if the string is 
 *                 nul-terminated<footnoteref linkend="nul-unsafe"/>. 
543 544 545
 * @converter:     conversion descriptor from g_iconv_open()
 * @bytes_read:    location to store the number of bytes in the
 *                 input string that were successfully converted, or %NULL.
Matthias Clasen's avatar
Matthias Clasen committed
546
 *                 Even if the conversion was successful, this may be 
547
 *                 less than @len if there were partial characters
548
 *                 at the end of the input. If the error
549 550
 *                 #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
 *                 stored will the byte offset after the last valid
551
 *                 input sequence.
552
 * @bytes_written: the number of bytes stored in the output buffer (not 
Matthias Clasen's avatar
Matthias Clasen committed
553
 *                 including the terminating nul).
554 555 556
 * @error:         location to store the error occuring, or %NULL to ignore
 *                 errors. Any of the errors in #GConvertError may occur.
 *
557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572
 * Converts a string from one character set to another. 
 * 
 * Note that you should use g_iconv() for streaming 
 * conversions<footnote id="streaming-state">
 *  <para>
 * Despite the fact that @byes_read can return information about partial 
 * characters, the <literal>g_convert_...</literal> functions
 * are not generally suitable for streaming. If the underlying converter 
 * being used maintains internal state, then this won't be preserved 
 * across successive calls to g_convert(), g_convert_with_iconv() or 
 * g_convert_with_fallback(). (An example of this is the GNU C converter 
 * for CP1255 which does not emit a base character until it knows that 
 * the next character is not a mark that could combine with the base 
 * character.)
 *  </para>
 * </footnote>. 
573 574
 *
 * Return value: If the conversion was successful, a newly allocated
575 576
 *               nul-terminated string, which must be freed with
 *               g_free(). Otherwise %NULL and @error will be set.
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
 **/
gchar*
g_convert_with_iconv (const gchar *str,
		      gssize       len,
		      GIConv       converter,
		      gsize       *bytes_read, 
		      gsize       *bytes_written, 
		      GError     **error)
{
  gchar *dest;
  gchar *outp;
  const gchar *p;
  gsize inbytes_remaining;
  gsize outbytes_remaining;
  gsize err;
  gsize outbuf_size;
  gboolean have_error = FALSE;
594
  gboolean done = FALSE;
595
  gboolean reset = FALSE;
596 597 598
  
  g_return_val_if_fail (converter != (GIConv) -1, NULL);
     
599 600 601 602 603
  if (len < 0)
    len = strlen (str);

  p = str;
  inbytes_remaining = len;
604
  outbuf_size = len + NUL_TERMINATOR_LENGTH;
605
  
606
  outbytes_remaining = outbuf_size - NUL_TERMINATOR_LENGTH;
607 608
  outp = dest = g_malloc (outbuf_size);

609
  while (!done && !have_error)
610
    {
611 612 613 614
      if (reset)
        err = g_iconv (converter, NULL, &inbytes_remaining, &outp, &outbytes_remaining);
      else
        err = g_iconv (converter, (char **)&p, &inbytes_remaining, &outp, &outbytes_remaining);
615

616
      if (err == (gsize) -1)
617
	{
618 619 620 621
	  switch (errno)
	    {
	    case EINVAL:
	      /* Incomplete text, do not report an error */
622
	      done = TRUE;
623 624 625
	      break;
	    case E2BIG:
	      {
626
		gsize used = outp - dest;
627
		
628 629 630 631
		outbuf_size *= 2;
		dest = g_realloc (dest, outbuf_size);
		
		outp = dest + used;
632
		outbytes_remaining = outbuf_size - used - NUL_TERMINATOR_LENGTH;
633 634 635 636
	      }
	      break;
	    case EILSEQ:
	      if (error)
637 638
		g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
                                     _("Invalid byte sequence in conversion input"));
639 640 641
	      have_error = TRUE;
	      break;
	    default:
Christian Persch's avatar
Christian Persch committed
642 643 644 645 646 647 648
              {
                int errsv = errno;
                
                g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
                             _("Error during conversion: %s"),
                             g_strerror (errsv));
              }
649 650 651 652 653 654
	      have_error = TRUE;
	      break;
	    }
	}
      else 
	{
655
	  if (!reset)
656 657
	    {
	      /* call g_iconv with NULL inbuf to cleanup shift state */
658
	      reset = TRUE;
659 660 661 662
	      inbytes_remaining = 0;
	    }
	  else
	    done = TRUE;
663 664 665
	}
    }

666
  memset (outp, 0, NUL_TERMINATOR_LENGTH);
667 668 669
  
  if (bytes_read)
    *bytes_read = p - str;
670 671 672 673
  else
    {
      if ((p - str) != len) 
	{
Havoc Pennington's avatar
Havoc Pennington committed
674 675
          if (!have_error)
            {
676
	      if (error)
677 678
		g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_PARTIAL_INPUT,
                                     _("Partial character sequence at end of input"));
Havoc Pennington's avatar
Havoc Pennington committed
679 680
              have_error = TRUE;
            }
681 682
	}
    }
683 684 685 686 687 688 689 690 691 692 693 694 695

  if (bytes_written)
    *bytes_written = outp - dest;	/* Doesn't include '\0' */

  if (have_error)
    {
      g_free (dest);
      return NULL;
    }
  else
    return dest;
}

696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
/**
 * g_convert:
 * @str:           the string to convert
 * @len:           the length of the string, or -1 if the string is 
 *                 nul-terminated<footnote id="nul-unsafe">
                     <para>
                       Note that some encodings may allow nul bytes to 
                       occur inside strings. In that case, using -1 for 
                       the @len parameter is unsafe.
                     </para>
                   </footnote>. 
 * @to_codeset:    name of character set into which to convert @str
 * @from_codeset:  character set of @str.
 * @bytes_read:    location to store the number of bytes in the
 *                 input string that were successfully converted, or %NULL.
 *                 Even if the conversion was successful, this may be 
 *                 less than @len if there were partial characters
 *                 at the end of the input. If the error
 *                 #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
 *                 stored will the byte offset after the last valid
 *                 input sequence.
 * @bytes_written: the number of bytes stored in the output buffer (not 
 *                 including the terminating nul).
 * @error:         location to store the error occuring, or %NULL to ignore
 *                 errors. Any of the errors in #GConvertError may occur.
 *
 * Converts a string from one character set to another.
 *
 * Note that you should use g_iconv() for streaming 
 * conversions<footnoteref linkend="streaming-state"/>.
 *
 * Return value: If the conversion was successful, a newly allocated
728 729
 *               nul-terminated string, which must be freed with
 *               g_free(). Otherwise %NULL and @error will be set.
730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
 **/
gchar*
g_convert (const gchar *str,
           gssize       len,  
           const gchar *to_codeset,
           const gchar *from_codeset,
           gsize       *bytes_read, 
	   gsize       *bytes_written, 
	   GError     **error)
{
  gchar *res;
  GIConv cd;

  g_return_val_if_fail (str != NULL, NULL);
  g_return_val_if_fail (to_codeset != NULL, NULL);
  g_return_val_if_fail (from_codeset != NULL, NULL);
  
  cd = open_converter (to_codeset, from_codeset, error);

  if (cd == (GIConv) -1)
    {
      if (bytes_read)
        *bytes_read = 0;
      
      if (bytes_written)
        *bytes_written = 0;
      
      return NULL;
    }

  res = g_convert_with_iconv (str, len, cd,
			      bytes_read, bytes_written,
			      error);

  close_converter (cd);

  return res;
}

769 770 771
/**
 * g_convert_with_fallback:
 * @str:          the string to convert
772 773
 * @len:          the length of the string, or -1 if the string is 
 *                nul-terminated<footnoteref linkend="nul-unsafe"/>. 
774 775 776
 * @to_codeset:   name of character set into which to convert @str
 * @from_codeset: character set of @str.
 * @fallback:     UTF-8 string to use in place of character not
777 778 779 780
 *                present in the target encoding. (The string must be
 *                representable in the target encoding). 
                  If %NULL, characters not in the target encoding will 
                  be represented as Unicode escapes \uxxxx or \Uxxxxyyyy.
781 782
 * @bytes_read:   location to store the number of bytes in the
 *                input string that were successfully converted, or %NULL.
Matthias Clasen's avatar
Matthias Clasen committed
783
 *                Even if the conversion was successful, this may be 
784
 *                less than @len if there were partial characters
785
 *                at the end of the input.
786
 * @bytes_written: the number of bytes stored in the output buffer (not 
Matthias Clasen's avatar
Matthias Clasen committed
787
 *                including the terminating nul).
788 789 790
 * @error:        location to store the error occuring, or %NULL to ignore
 *                errors. Any of the errors in #GConvertError may occur.
 *
Matthias Clasen's avatar
Matthias Clasen committed
791
 * Converts a string from one character set to another, possibly
792 793 794
 * including fallback sequences for characters not representable
 * in the output. Note that it is not guaranteed that the specification
 * for the fallback sequences in @fallback will be honored. Some
795
 * systems may do an approximate conversion from @from_codeset
796
 * to @to_codeset in their iconv() functions, 
Owen Taylor's avatar
Owen Taylor committed
797
 * in which case GLib will simply return that approximate conversion.
798
 *
799 800 801
 * Note that you should use g_iconv() for streaming 
 * conversions<footnoteref linkend="streaming-state"/>.
 *
802
 * Return value: If the conversion was successful, a newly allocated
803 804
 *               nul-terminated string, which must be freed with
 *               g_free(). Otherwise %NULL and @error will be set.
805 806 807
 **/
gchar*
g_convert_with_fallback (const gchar *str,
808
			 gssize       len,    
809 810
			 const gchar *to_codeset,
			 const gchar *from_codeset,
811
			 const gchar *fallback,
812 813
			 gsize       *bytes_read,
			 gsize       *bytes_written,
814 815 816 817 818 819 820
			 GError     **error)
{
  gchar *utf8;
  gchar *dest;
  gchar *outp;
  const gchar *insert_str = NULL;
  const gchar *p;
821
  gsize inbytes_remaining;   
822
  const gchar *save_p = NULL;
823 824 825
  gsize save_inbytes = 0;
  gsize outbytes_remaining; 
  gsize err;
826
  GIConv cd;
827
  gsize outbuf_size;
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
  gboolean have_error = FALSE;
  gboolean done = FALSE;

  GError *local_error = NULL;
  
  g_return_val_if_fail (str != NULL, NULL);
  g_return_val_if_fail (to_codeset != NULL, NULL);
  g_return_val_if_fail (from_codeset != NULL, NULL);
     
  if (len < 0)
    len = strlen (str);
  
  /* Try an exact conversion; we only proceed if this fails
   * due to an illegal sequence in the input string.
   */
  dest = g_convert (str, len, to_codeset, from_codeset, 
		    bytes_read, bytes_written, &local_error);
  if (!local_error)
    return dest;

  if (!g_error_matches (local_error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE))
    {
      g_propagate_error (error, local_error);
      return NULL;
    }
  else
    g_error_free (local_error);

Havoc Pennington's avatar
Havoc Pennington committed
856 857
  local_error = NULL;
  
858 859 860 861
  /* No go; to proceed, we need a converter from "UTF-8" to
   * to_codeset, and the string as UTF-8.
   */
  cd = open_converter (to_codeset, "UTF-8", error);
862
  if (cd == (GIConv) -1)
863 864 865 866 867 868 869 870 871 872 873 874 875
    {
      if (bytes_read)
        *bytes_read = 0;
      
      if (bytes_written)
        *bytes_written = 0;
      
      return NULL;
    }

  utf8 = g_convert (str, len, "UTF-8", from_codeset, 
		    bytes_read, &inbytes_remaining, error);
  if (!utf8)
876
    {
877
      close_converter (cd);
878 879 880 881
      if (bytes_written)
        *bytes_written = 0;
      return NULL;
    }
882 883 884 885 886 887 888 889 890 891

  /* Now the heart of the code. We loop through the UTF-8 string, and
   * whenever we hit an offending character, we form fallback, convert
   * the fallback to the target codeset, and then go back to
   * converting the original string after finishing with the fallback.
   *
   * The variables save_p and save_inbytes store the input state
   * for the original string while we are converting the fallback
   */
  p = utf8;
892

893 894
  outbuf_size = len + NUL_TERMINATOR_LENGTH;
  outbytes_remaining = outbuf_size - NUL_TERMINATOR_LENGTH;
895 896 897 898
  outp = dest = g_malloc (outbuf_size);

  while (!done && !have_error)
    {
899
      gsize inbytes_tmp = inbytes_remaining;
900
      err = g_iconv (cd, (char **)&p, &inbytes_tmp, &outp, &outbytes_remaining);
901
      inbytes_remaining = inbytes_tmp;
902

903
      if (err == (gsize) -1)
904 905 906 907 908 909 910 911
	{
	  switch (errno)
	    {
	    case EINVAL:
	      g_assert_not_reached();
	      break;
	    case E2BIG:
	      {
912
		gsize used = outp - dest;
913

914 915 916 917
		outbuf_size *= 2;
		dest = g_realloc (dest, outbuf_size);
		
		outp = dest + used;
918
		outbytes_remaining = outbuf_size - used - NUL_TERMINATOR_LENGTH;
919 920 921 922 923 924 925 926 927
		
		break;
	      }
	    case EILSEQ:
	      if (save_p)
		{
		  /* Error converting fallback string - fatal
		   */
		  g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
928
			       _("Cannot convert fallback '%s' to codeset '%s'"),
929 930 931 932
			       insert_str, to_codeset);
		  have_error = TRUE;
		  break;
		}
933
	      else if (p)
934 935 936 937
		{
		  if (!fallback)
		    { 
		      gunichar ch = g_utf8_get_char (p);
938
		      insert_str = g_strdup_printf (ch < 0x10000 ? "\\u%04x" : "\\U%08x",
939 940 941 942 943 944 945 946 947
						    ch);
		    }
		  else
		    insert_str = fallback;
		  
		  save_p = g_utf8_next_char (p);
		  save_inbytes = inbytes_remaining - (save_p - p);
		  p = insert_str;
		  inbytes_remaining = strlen (p);
948
		  break;
949
		}
950
	      /* fall thru if p is NULL */
951
	    default:
Christian Persch's avatar
Christian Persch committed
952 953 954 955 956 957 958 959
              {
                int errsv = errno;

                g_set_error (error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
                             _("Error during conversion: %s"),
                             g_strerror (errsv));
              }

960 961 962 963 964 965 966 967 968 969 970 971 972 973
	      have_error = TRUE;
	      break;
	    }
	}
      else
	{
	  if (save_p)
	    {
	      if (!fallback)
		g_free ((gchar *)insert_str);
	      p = save_p;
	      inbytes_remaining = save_inbytes;
	      save_p = NULL;
	    }
974 975 976 977 978 979
	  else if (p)
	    {
	      /* call g_iconv with NULL inbuf to cleanup shift state */
	      p = NULL;
	      inbytes_remaining = 0;
	    }
980 981 982 983 984 985 986
	  else
	    done = TRUE;
	}
    }

  /* Cleanup
   */
987
  memset (outp, 0, NUL_TERMINATOR_LENGTH);
988
  
989
  close_converter (cd);
990 991

  if (bytes_written)
992
    *bytes_written = outp - dest;	/* Doesn't include '\0' */
993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005

  g_free (utf8);

  if (have_error)
    {
      if (save_p && !fallback)
	g_free ((gchar *)insert_str);
      g_free (dest);
      return NULL;
    }
  else
    return dest;
}
1006 1007 1008 1009

/*
 * g_locale_to_utf8
 *
1010 1011 1012
 * 
 */

1013 1014 1015 1016
static gchar *
strdup_len (const gchar *string,
	    gssize       len,
	    gsize       *bytes_written,
1017 1018
	    gsize       *bytes_read,
	    GError      **error)
1019 1020 1021 1022
	 
{
  gsize real_len;

1023
  if (!g_utf8_validate (string, len, NULL))
1024 1025 1026 1027 1028 1029
    {
      if (bytes_read)
	*bytes_read = 0;
      if (bytes_written)
	*bytes_written = 0;

1030 1031
      g_set_error_literal (error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
                           _("Invalid byte sequence in conversion input"));
1032 1033 1034
      return NULL;
    }
  
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
  if (len < 0)
    real_len = strlen (string);
  else
    {
      real_len = 0;
      
      while (real_len < len && string[real_len])
	real_len++;
    }
  
  if (bytes_read)
    *bytes_read = real_len;
  if (bytes_written)
    *bytes_written = real_len;

  return g_strndup (string, real_len);
}

1053 1054
/**
 * g_locale_to_utf8:
Tor Lillqvist's avatar
Tor Lillqvist committed
1055 1056
 * @opsysstring:   a string in the encoding of the current locale. On Windows
 *                 this means the system codepage.
1057
 * @len:           the length of the string, or -1 if the string is
1058
 *                 nul-terminated<footnoteref linkend="nul-unsafe"/>. 
1059 1060
 * @bytes_read:    location to store the number of bytes in the
 *                 input string that were successfully converted, or %NULL.
Matthias Clasen's avatar
Matthias Clasen committed
1061
 *                 Even if the conversion was successful, this may be 
1062
 *                 less than @len if there were partial characters
1063
 *                 at the end of the input. If the error
1064 1065
 *                 #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
 *                 stored will the byte offset after the last valid
1066
 *                 input sequence.
1067
 * @bytes_written: the number of bytes stored in the output buffer (not 
Matthias Clasen's avatar
Matthias Clasen committed
1068
 *                 including the terminating nul).
1069
 * @error:         location to store the error occuring, or %NULL to ignore
1070 1071
 *                 errors. Any of the errors in #GConvertError may occur.
 * 
1072 1073
 * Converts a string which is in the encoding used for strings by
 * the C runtime (usually the same as that used by the operating
1074 1075
 * system) in the <link linkend="setlocale">current locale</link> into a
 * UTF-8 string.
1076 1077 1078
 * 
 * Return value: The converted string, or %NULL on an error.
 **/
1079
gchar *
1080
g_locale_to_utf8 (const gchar  *opsysstring,
1081 1082 1083
		  gssize        len,            
		  gsize        *bytes_read,    
		  gsize        *bytes_written,
1084
		  GError      **error)
1085
{
1086
  const char *charset;
1087 1088

  if (g_get_charset (&charset))
1089
    return strdup_len (opsysstring, len, bytes_read, bytes_written, error);
1090 1091 1092
  else
    return g_convert (opsysstring, len, 
		      "UTF-8", charset, bytes_read, bytes_written, error);
1093 1094
}

1095 1096 1097 1098
/**
 * g_locale_from_utf8:
 * @utf8string:    a UTF-8 encoded string 
 * @len:           the length of the string, or -1 if the string is
1099
 *                 nul-terminated<footnoteref linkend="nul-unsafe"/>. 
1100 1101
 * @bytes_read:    location to store the number of bytes in the
 *                 input string that were successfully converted, or %NULL.
Matthias Clasen's avatar
Matthias Clasen committed
1102
 *                 Even if the conversion was successful, this may be 
1103
 *                 less than @len if there were partial characters
1104
 *                 at the end of the input. If the error
1105 1106
 *                 #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
 *                 stored will the byte offset after the last valid
1107
 *                 input sequence.
1108
 * @bytes_written: the number of bytes stored in the output buffer (not 
Matthias Clasen's avatar
Matthias Clasen committed
1109
 *                 including the terminating nul).
1110
 * @error:         location to store the error occuring, or %NULL to ignore
1111 1112 1113 1114
 *                 errors. Any of the errors in #GConvertError may occur.
 * 
 * Converts a string from UTF-8 to the encoding used for strings by
 * the C runtime (usually the same as that used by the operating
1115 1116
 * system) in the <link linkend="setlocale">current locale</link>. On
 * Windows this means the system codepage.
1117 1118 1119
 * 
 * Return value: The converted string, or %NULL on an error.
 **/
1120
gchar *
1121
g_locale_from_utf8 (const gchar *utf8string,
1122 1123 1124
		    gssize       len,            
		    gsize       *bytes_read,    
		    gsize       *bytes_written,
1125
		    GError     **error)
1126
{
1127
  const gchar *charset;
1128 1129

  if (g_get_charset (&charset))
1130
    return strdup_len (utf8string, len, bytes_read, bytes_written, error);
1131 1132 1133
  else
    return g_convert (utf8string, len,
		      charset, "UTF-8", bytes_read, bytes_written, error);
1134 1135
}

1136
#ifndef G_PLATFORM_WIN32
1137

1138 1139 1140 1141 1142
typedef struct _GFilenameCharsetCache GFilenameCharsetCache;

struct _GFilenameCharsetCache {
  gboolean is_utf8;
  gchar *charset;
1143
  gchar **filename_charsets;
1144 1145 1146 1147 1148 1149 1150
};

static void
filename_charset_cache_free (gpointer data)
{
  GFilenameCharsetCache *cache = data;
  g_free (cache->charset);
1151
  g_strfreev (cache->filename_charsets);
1152 1153 1154
  g_free (cache);
}

Matthias Clasen's avatar
Matthias Clasen committed
1155
/**
1156 1157
 * g_get_filename_charsets:
 * @charsets: return location for the %NULL-terminated list of encoding names
1158
 *
1159 1160 1161
 * Determines the preferred character sets used for filenames.
 * The first character set from the @charsets is the filename encoding, the
 * subsequent character sets are used when trying to generate a displayable
Tor Lillqvist's avatar
Tor Lillqvist committed
1162
 * representation of a filename, see g_filename_display_name().
1163
 *
1164 1165 1166 1167 1168
 * On Unix, the character sets are determined by consulting the
 * environment variables <envar>G_FILENAME_ENCODING</envar> and
 * <envar>G_BROKEN_FILENAMES</envar>. On Windows, the character set
 * used in the GLib API is always UTF-8 and said environment variables
 * have no effect.
1169
 *
1170 1171 1172 1173 1174 1175 1176
 * <envar>G_FILENAME_ENCODING</envar> may be set to a comma-separated list 
 * of character set names. The special token "&commat;locale" is taken to 
 * mean the character set for the <link linkend="setlocale">current 
 * locale</link>. If <envar>G_FILENAME_ENCODING</envar> is not set, but 
 * <envar>G_BROKEN_FILENAMES</envar> is, the character set of the current 
 * locale is taken as the filename encoding. If neither environment variable 
 * is set, UTF-8 is taken as the filename encoding, but the character
1177 1178 1179
 * set of the current locale is also put in the list of encodings.
 *
 * The returned @charsets belong to GLib and must not be freed.
1180 1181
 *
 * Note that on Unix, regardless of the locale character set or
1182 1183
 * <envar>G_FILENAME_ENCODING</envar> value, the actual file names present 
 * on a system might be in any random encoding or just gibberish.
1184
 *
1185 1186 1187
 * Return value: %TRUE if the filename encoding is UTF-8.
 * 
 * Since: 2.6
1188
 */
1189 1190
gboolean
g_get_filename_charsets (G_CONST_RETURN gchar ***filename_charsets)
1191
{
1192 1193 1194
  static GStaticPrivate cache_private = G_STATIC_PRIVATE_INIT;
  GFilenameCharsetCache *cache = g_static_private_get (&cache_private);
  const gchar *charset;
1195

1196
  if (!cache)
1197
    {
1198 1199 1200 1201 1202 1203 1204 1205 1206
      cache = g_new0 (GFilenameCharsetCache, 1);
      g_static_private_set (&cache_private, cache, filename_charset_cache_free);
    }

  g_get_charset (&charset);

  if (!(cache->charset && strcmp (cache->charset, charset) == 0))
    {
      const gchar *new_charset;
1207 1208
      gchar *p;
      gint i;
1209

1210
      g_free (cache->charset);
1211
      g_strfreev (cache->filename_charsets);
1212
      cache->charset = g_strdup (charset);
1213 1214
      
      p = getenv ("G_FILENAME_ENCODING");
1215
      if (p != NULL && p[0] != '\0') 
1216
	{
1217 1218
	  cache->filename_charsets = g_strsplit (p, ",", 0);
	  cache->is_utf8 = (strcmp (cache->filename_charsets[0], "UTF-8") == 0);
1219

1220
	  for (i = 0; cache->filename_charsets[i]; i++)
1221
	    {
1222 1223 1224 1225 1226 1227
	      if (strcmp ("@locale", cache->filename_charsets[i]) == 0)
		{
		  g_get_charset (&new_charset);
		  g_free (cache->filename_charsets[i]);
		  cache->filename_charsets[i] = g_strdup (new_charset);
		}
1228 1229 1230
	    }
	}
      else if (getenv ("G_BROKEN_FILENAMES") != NULL)
1231
	{
1232
	  cache->filename_charsets = g_new0 (gchar *, 2);
1233
	  cache->is_utf8 = g_get_charset (&new_charset);
1234
	  cache->filename_charsets[0] = g_strdup (new_charset);
1235
	}
1236 1237
      else 
	{
1238
	  cache->filename_charsets = g_new0 (gchar *, 3);
1239
	  cache->is_utf8 = TRUE;
1240 1241 1242
	  cache->filename_charsets[0] = g_strdup ("UTF-8");
	  if (!g_get_charset (&new_charset))
	    cache->filename_charsets[1] = g_strdup (new_charset);
1243 1244
	}
    }
1245

1246 1247
  if (filename_charsets)
    *filename_charsets = (const gchar **)cache->filename_charsets;
1248

1249
  return cache->is_utf8;
1250
}
1251

1252
#else /* G_PLATFORM_WIN32 */
1253

1254 1255
gboolean
g_get_filename_charsets (G_CONST_RETURN gchar ***filename_charsets) 
1256
{
1257
  static const gchar *charsets[] = {
1258 1259 1260 1261
    "UTF-8",
    NULL
  };

1262 1263
#ifdef G_OS_WIN32
  /* On Windows GLib pretends that the filename charset is UTF-8 */
1264 1265 1266
  if (filename_charsets)
    *filename_charsets = charsets;

1267 1268
  return TRUE;
#else
1269 1270
  gboolean result;

1271
  /* Cygwin works like before */
1272 1273 1274 1275 1276 1277
  result = g_get_charset (&(charsets[0]));

  if (filename_charsets)
    *filename_charsets = charsets;

  return result;
1278 1279 1280
#endif
}

1281
#endif /* G_PLATFORM_WIN32 */
1282 1283

static gboolean
1284
get_filename_charset (const gchar **filename_charset)
1285
{
1286 1287 1288 1289
  const gchar **charsets;
  gboolean is_utf8;
  
  is_utf8 = g_get_filename_charsets (&charsets);
1290

1291 1292 1293 1294 1295
  if (filename_charset)
    *filename_charset = charsets[0];
  
  return is_utf8;
}
1296 1297 1298 1299 1300

/* This is called from g_thread_init(). It's used to
 * initialize some static data in a threadsafe way.
 */
void 
1301
_g_convert_thread_init (void)
1302
{
1303
  const gchar **dummy;
Matthias Clasen's avatar
Matthias Clasen committed
1304
  (void) g_get_filename_charsets (&dummy);
1305 1306
}

1307 1308 1309 1310
/**
 * g_filename_to_utf8:
 * @opsysstring:   a string in the encoding for filenames
 * @len:           the length of the string, or -1 if the string is
1311
 *                 nul-terminated<footnoteref linkend="nul-unsafe"/>. 
1312 1313
 * @bytes_read:    location to store the number of bytes in the
 *                 input string that were successfully converted, or %NULL.
Matthias Clasen's avatar
Matthias Clasen committed
1314
 *                 Even if the conversion was successful, this may be 
1315
 *                 less than @len if there were partial characters
1316
 *                 at the end of the input. If the error
1317 1318
 *                 #G_CONVERT_ERROR_ILLEGAL_SEQUENCE occurs, the value
 *                 stored will the byte offset after the last valid
1319
 *                 input sequence.
1320
 * @bytes_written: the number of bytes stored in the output buffer (not 
Matthias Clasen's avatar
Matthias Clasen committed
1321
 *                 including the terminating nul).
1322
 * @error:         location to store the error occuring, or %NULL to ignore