1/* Convert text in given files from the specified from-set to the to-set.
2 Copyright (C) 1998-2016 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published
8 by the Free Software Foundation; version 2 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <http://www.gnu.org/licenses/>. */
18
19#include <argp.h>
20#include <assert.h>
21#include <ctype.h>
22#include <errno.h>
23#include <error.h>
24#include <fcntl.h>
25#include <iconv.h>
26#include <langinfo.h>
27#include <locale.h>
28#include <search.h>
29#include <stdbool.h>
30#include <stdio.h>
31#include <stdlib.h>
32#include <string.h>
33#include <unistd.h>
34#include <libintl.h>
35#ifdef _POSIX_MAPPED_FILES
36# include <sys/mman.h>
37#endif
38#include <charmap.h>
39#include <gconv_int.h>
40#include "iconv_prog.h"
41#include "iconvconfig.h"
42
43/* Get libc version number. */
44#include "../version.h"
45
46#define PACKAGE _libc_intl_domainname
47
48
49/* Name and version of program. */
50static void print_version (FILE *stream, struct argp_state *state);
51void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
52
53#define OPT_VERBOSE 1000
54#define OPT_LIST 'l'
55
56/* Definitions of arguments for argp functions. */
57static const struct argp_option options[] =
58{
59 { NULL, 0, NULL, 0, N_("Input/Output format specification:") },
60 { "from-code", 'f', N_("NAME"), 0, N_("encoding of original text") },
61 { "to-code", 't', N_("NAME"), 0, N_("encoding for output") },
62 { NULL, 0, NULL, 0, N_("Information:") },
63 { "list", 'l', NULL, 0, N_("list all known coded character sets") },
64 { NULL, 0, NULL, 0, N_("Output control:") },
65 { NULL, 'c', NULL, 0, N_("omit invalid characters from output") },
66 { "output", 'o', N_("FILE"), 0, N_("output file") },
67 { "silent", 's', NULL, 0, N_("suppress warnings") },
68 { "verbose", OPT_VERBOSE, NULL, 0, N_("print progress information") },
69 { NULL, 0, NULL, 0, NULL }
70};
71
72/* Short description of program. */
73static const char doc[] = N_("\
74Convert encoding of given files from one encoding to another.");
75
76/* Strings for arguments in help texts. */
77static const char args_doc[] = N_("[FILE...]");
78
79/* Prototype for option handler. */
80static error_t parse_opt (int key, char *arg, struct argp_state *state);
81
82/* Function to print some extra text in the help message. */
83static char *more_help (int key, const char *text, void *input);
84
85/* Data structure to communicate with argp functions. */
86static struct argp argp =
87{
88 options, parse_opt, args_doc, doc, NULL, more_help
89};
90
91/* Code sets to convert from and to respectively. An empty string as the
92 default causes the 'iconv_open' function to look up the charset of the
93 currently selected locale and use it. */
94static const char *from_code = "";
95static const char *to_code = "";
96
97/* File to write output to. If NULL write to stdout. */
98static const char *output_file;
99
100/* Nonzero if verbose ouput is wanted. */
101int verbose;
102
103/* Nonzero if list of all coded character sets is wanted. */
104static int list;
105
106/* If nonzero omit invalid character from output. */
107int omit_invalid;
108
109/* Prototypes for the functions doing the actual work. */
110static int process_block (iconv_t cd, char *addr, size_t len, FILE **output,
111 const char *output_file);
112static int process_fd (iconv_t cd, int fd, FILE **output,
113 const char *output_file);
114static int process_file (iconv_t cd, FILE *input, FILE **output,
115 const char *output_file);
116static void print_known_names (void) internal_function;
117
118
119int
120main (int argc, char *argv[])
121{
122 int status = EXIT_SUCCESS;
123 int remaining;
124 iconv_t cd;
125 const char *orig_to_code;
126 struct charmap_t *from_charmap = NULL;
127 struct charmap_t *to_charmap = NULL;
128
129 /* Set locale via LC_ALL. */
130 setlocale (LC_ALL, "");
131
132 /* Set the text message domain. */
133 textdomain (_libc_intl_domainname);
134
135 /* Parse and process arguments. */
136 argp_parse (&argp, argc, argv, 0, &remaining, NULL);
137
138 /* List all coded character sets if wanted. */
139 if (list)
140 {
141 print_known_names ();
142 exit (EXIT_SUCCESS);
143 }
144
145 /* If we have to ignore errors make sure we use the appropriate name for
146 the to-character-set. */
147 orig_to_code = to_code;
148 if (omit_invalid)
149 {
150 const char *errhand = strchrnul (to_code, '/');
151 int nslash = 2;
152 char *newp;
153 char *cp;
154
155 if (*errhand == '/')
156 {
157 --nslash;
158 errhand = strchrnul (errhand + 1, '/');
159
160 if (*errhand == '/')
161 {
162 --nslash;
163 errhand = strchr (errhand, '\0');
164 }
165 }
166
167 newp = (char *) alloca (errhand - to_code + nslash + 7 + 1);
168 cp = mempcpy (newp, to_code, errhand - to_code);
169 while (nslash-- > 0)
170 *cp++ = '/';
171 if (cp[-1] != '/')
172 *cp++ = ',';
173 memcpy (cp, "IGNORE", sizeof ("IGNORE"));
174
175 to_code = newp;
176 }
177
178 /* POSIX 1003.2b introduces a silly thing: the arguments to -t anf -f
179 can be file names of charmaps. In this case iconv will have to read
180 those charmaps and use them to do the conversion. But there are
181 holes in the specification. There is nothing said that if -f is a
182 charmap filename that -t must be, too. And vice versa. There is
183 also no word about the symbolic names used. What if they don't
184 match? */
185 if (strchr (from_code, '/') != NULL)
186 /* The from-name might be a charmap file name. Try reading the
187 file. */
188 from_charmap = charmap_read (from_code, /*0, 1*/1, 0, 0, 0);
189
190 if (strchr (orig_to_code, '/') != NULL)
191 /* The to-name might be a charmap file name. Try reading the
192 file. */
193 to_charmap = charmap_read (orig_to_code, /*0, 1,*/1, 0, 0, 0);
194
195
196 /* At this point we have to handle two cases. The first one is
197 where a charmap is used for the from- or to-charset, or both. We
198 handle this special since it is very different from the sane way of
199 doing things. The other case allows converting using the iconv()
200 function. */
201 if (from_charmap != NULL || to_charmap != NULL)
202 /* Construct the conversion table and do the conversion. */
203 status = charmap_conversion (from_code, from_charmap, to_code, to_charmap,
204 argc, remaining, argv, output_file);
205 else
206 {
207 /* Let's see whether we have these coded character sets. */
208 cd = iconv_open (to_code, from_code);
209 if (cd == (iconv_t) -1)
210 {
211 if (errno == EINVAL)
212 {
213 /* Try to be nice with the user and tell her which of the
214 two encoding names is wrong. This is possible because
215 all supported encodings can be converted from/to Unicode,
216 in other words, because the graph of encodings is
217 connected. */
218 bool from_wrong =
219 (iconv_open ("UTF-8", from_code) == (iconv_t) -1
220 && errno == EINVAL);
221 bool to_wrong =
222 (iconv_open (to_code, "UTF-8") == (iconv_t) -1
223 && errno == EINVAL);
224 const char *from_pretty =
225 (from_code[0] ? from_code : nl_langinfo (CODESET));
226 const char *to_pretty =
227 (orig_to_code[0] ? orig_to_code : nl_langinfo (CODESET));
228
229 if (from_wrong)
230 {
231 if (to_wrong)
232 error (0, 0,
233 _("\
234conversions from `%s' and to `%s' are not supported"),
235 from_pretty, to_pretty);
236 else
237 error (0, 0,
238 _("conversion from `%s' is not supported"),
239 from_pretty);
240 }
241 else
242 {
243 if (to_wrong)
244 error (0, 0,
245 _("conversion to `%s' is not supported"),
246 to_pretty);
247 else
248 error (0, 0,
249 _("conversion from `%s' to `%s' is not supported"),
250 from_pretty, to_pretty);
251 }
252
253 argp_help (&argp, stderr, ARGP_HELP_SEE,
254 program_invocation_short_name);
255 exit (1);
256 }
257 else
258 error (EXIT_FAILURE, errno,
259 _("failed to start conversion processing"));
260 }
261
262 /* The output file. Will be opened when we are ready to produce
263 output. */
264 FILE *output = NULL;
265
266 /* Now process the remaining files. Write them to stdout or the file
267 specified with the `-o' parameter. If we have no file given as
268 the parameter process all from stdin. */
269 if (remaining == argc)
270 {
271 if (process_file (cd, stdin, &output, output_file) != 0)
272 status = EXIT_FAILURE;
273 }
274 else
275 do
276 {
277#ifdef _POSIX_MAPPED_FILES
278 struct stat64 st;
279 char *addr;
280#endif
281 int fd, ret;
282
283 if (verbose)
284 fprintf (stderr, "%s:\n", argv[remaining]);
285 if (strcmp (argv[remaining], "-") == 0)
286 fd = 0;
287 else
288 {
289 fd = open (argv[remaining], O_RDONLY);
290
291 if (fd == -1)
292 {
293 error (0, errno, _("cannot open input file `%s'"),
294 argv[remaining]);
295 status = EXIT_FAILURE;
296 continue;
297 }
298 }
299
300#ifdef _POSIX_MAPPED_FILES
301 /* We have possibilities for reading the input file. First try
302 to mmap() it since this will provide the fastest solution. */
303 if (fstat64 (fd, &st) == 0
304 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
305 fd, 0)) != MAP_FAILED))
306 {
307 /* Yes, we can use mmap(). The descriptor is not needed
308 anymore. */
309 if (close (fd) != 0)
310 error (EXIT_FAILURE, errno,
311 _("error while closing input `%s'"),
312 argv[remaining]);
313
314 ret = process_block (cd, addr, st.st_size, &output,
315 output_file);
316
317 /* We don't need the input data anymore. */
318 munmap ((void *) addr, st.st_size);
319
320 if (ret != 0)
321 {
322 status = EXIT_FAILURE;
323
324 if (ret < 0)
325 /* We cannot go on with producing output since it might
326 lead to problem because the last output might leave
327 the output stream in an undefined state. */
328 break;
329 }
330 }
331 else
332#endif /* _POSIX_MAPPED_FILES */
333 {
334 /* Read the file in pieces. */
335 ret = process_fd (cd, fd, &output, output_file);
336
337 /* Now close the file. */
338 close (fd);
339
340 if (ret != 0)
341 {
342 /* Something went wrong. */
343 status = EXIT_FAILURE;
344
345 if (ret < 0)
346 /* We cannot go on with producing output since it might
347 lead to problem because the last output might leave
348 the output stream in an undefined state. */
349 break;
350 }
351 }
352 }
353 while (++remaining < argc);
354
355 /* Close the output file now. */
356 if (output != NULL && fclose (output))
357 error (EXIT_FAILURE, errno, _("error while closing output file"));
358 }
359
360 return status;
361}
362
363
364/* Handle program arguments. */
365static error_t
366parse_opt (int key, char *arg, struct argp_state *state)
367{
368 switch (key)
369 {
370 case 'f':
371 from_code = arg;
372 break;
373 case 't':
374 to_code = arg;
375 break;
376 case 'o':
377 output_file = arg;
378 break;
379 case 's':
380 /* Nothing, for now at least. We are not giving out any information
381 about missing character or so. */
382 break;
383 case 'c':
384 /* Omit invalid characters from output. */
385 omit_invalid = 1;
386 break;
387 case OPT_VERBOSE:
388 verbose = 1;
389 break;
390 case OPT_LIST:
391 list = 1;
392 break;
393 default:
394 return ARGP_ERR_UNKNOWN;
395 }
396 return 0;
397}
398
399
400static char *
401more_help (int key, const char *text, void *input)
402{
403 char *tp = NULL;
404 switch (key)
405 {
406 case ARGP_KEY_HELP_EXTRA:
407 /* We print some extra information. */
408 if (asprintf (&tp, gettext ("\
409For bug reporting instructions, please see:\n\
410%s.\n"), REPORT_BUGS_TO) < 0)
411 return NULL;
412 return tp;
413 default:
414 break;
415 }
416 return (char *) text;
417}
418
419
420/* Print the version information. */
421static void
422print_version (FILE *stream, struct argp_state *state)
423{
424 fprintf (stream, "iconv %s%s\n", PKGVERSION, VERSION);
425 fprintf (stream, gettext ("\
426Copyright (C) %s Free Software Foundation, Inc.\n\
427This is free software; see the source for copying conditions. There is NO\n\
428warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
429"), "2016");
430 fprintf (stream, gettext ("Written by %s.\n"), "Ulrich Drepper");
431}
432
433
434static int
435write_output (const char *outbuf, const char *outptr, FILE **output,
436 const char *output_file)
437{
438 /* We have something to write out. */
439 int errno_save = errno;
440
441 if (*output == NULL)
442 {
443 /* Determine output file. */
444 if (output_file != NULL && strcmp (output_file, "-") != 0)
445 {
446 *output = fopen (output_file, "w");
447 if (*output == NULL)
448 error (EXIT_FAILURE, errno, _("cannot open output file"));
449 }
450 else
451 *output = stdout;
452 }
453
454 if (fwrite (outbuf, 1, outptr - outbuf, *output) < (size_t) (outptr - outbuf)
455 || ferror (*output))
456 {
457 /* Error occurred while printing the result. */
458 error (0, 0, _("\
459conversion stopped due to problem in writing the output"));
460 return -1;
461 }
462
463 errno = errno_save;
464
465 return 0;
466}
467
468
469static int
470process_block (iconv_t cd, char *addr, size_t len, FILE **output,
471 const char *output_file)
472{
473#define OUTBUF_SIZE 32768
474 const char *start = addr;
475 char outbuf[OUTBUF_SIZE];
476 char *outptr;
477 size_t outlen;
478 size_t n;
479 int ret = 0;
480
481 while (len > 0)
482 {
483 outptr = outbuf;
484 outlen = OUTBUF_SIZE;
485 n = iconv (cd, &addr, &len, &outptr, &outlen);
486
487 if (n == (size_t) -1 && omit_invalid && errno == EILSEQ)
488 {
489 ret = 1;
490 if (len == 0)
491 n = 0;
492 else
493 errno = E2BIG;
494 }
495
496 if (outptr != outbuf)
497 {
498 ret = write_output (outbuf, outptr, output, output_file);
499 if (ret != 0)
500 break;
501 }
502
503 if (n != (size_t) -1)
504 {
505 /* All the input test is processed. For state-dependent
506 character sets we have to flush the state now. */
507 outptr = outbuf;
508 outlen = OUTBUF_SIZE;
509 n = iconv (cd, NULL, NULL, &outptr, &outlen);
510
511 if (outptr != outbuf)
512 {
513 ret = write_output (outbuf, outptr, output, output_file);
514 if (ret != 0)
515 break;
516 }
517
518 if (n != (size_t) -1)
519 break;
520
521 if (omit_invalid && errno == EILSEQ)
522 {
523 ret = 1;
524 break;
525 }
526 }
527
528 if (errno != E2BIG)
529 {
530 /* iconv() ran into a problem. */
531 switch (errno)
532 {
533 case EILSEQ:
534 if (! omit_invalid)
535 error (0, 0, _("illegal input sequence at position %ld"),
536 (long int) (addr - start));
537 break;
538 case EINVAL:
539 error (0, 0, _("\
540incomplete character or shift sequence at end of buffer"));
541 break;
542 case EBADF:
543 error (0, 0, _("internal error (illegal descriptor)"));
544 break;
545 default:
546 error (0, 0, _("unknown iconv() error %d"), errno);
547 break;
548 }
549
550 return -1;
551 }
552 }
553
554 return ret;
555}
556
557
558static int
559process_fd (iconv_t cd, int fd, FILE **output, const char *output_file)
560{
561 /* we have a problem with reading from a desriptor since we must not
562 provide the iconv() function an incomplete character or shift
563 sequence at the end of the buffer. Since we have to deal with
564 arbitrary encodings we must read the whole text in a buffer and
565 process it in one step. */
566 static char *inbuf = NULL;
567 static size_t maxlen = 0;
568 char *inptr = NULL;
569 size_t actlen = 0;
570
571 while (actlen < maxlen)
572 {
573 ssize_t n = read (fd, inptr, maxlen - actlen);
574
575 if (n == 0)
576 /* No more text to read. */
577 break;
578
579 if (n == -1)
580 {
581 /* Error while reading. */
582 error (0, errno, _("error while reading the input"));
583 return -1;
584 }
585
586 inptr += n;
587 actlen += n;
588 }
589
590 if (actlen == maxlen)
591 while (1)
592 {
593 ssize_t n;
594 char *new_inbuf;
595
596 /* Increase the buffer. */
597 new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
598 if (new_inbuf == NULL)
599 {
600 error (0, errno, _("unable to allocate buffer for input"));
601 return -1;
602 }
603 inbuf = new_inbuf;
604 maxlen += 32768;
605 inptr = inbuf + actlen;
606
607 do
608 {
609 n = read (fd, inptr, maxlen - actlen);
610
611 if (n == 0)
612 /* No more text to read. */
613 break;
614
615 if (n == -1)
616 {
617 /* Error while reading. */
618 error (0, errno, _("error while reading the input"));
619 return -1;
620 }
621
622 inptr += n;
623 actlen += n;
624 }
625 while (actlen < maxlen);
626
627 if (n == 0)
628 /* Break again so we leave both loops. */
629 break;
630 }
631
632 /* Now we have all the input in the buffer. Process it in one run. */
633 return process_block (cd, inbuf, actlen, output, output_file);
634}
635
636
637static int
638process_file (iconv_t cd, FILE *input, FILE **output, const char *output_file)
639{
640 /* This should be safe since we use this function only for `stdin' and
641 we haven't read anything so far. */
642 return process_fd (cd, fileno (input), output, output_file);
643}
644
645
646/* Print all known character sets/encodings. */
647static void *printlist;
648static size_t column;
649static int not_first;
650
651static void
652insert_print_list (const void *nodep, VISIT value, int level)
653{
654 if (value == leaf || value == postorder)
655 {
656 const struct gconv_alias *s = *(const struct gconv_alias **) nodep;
657 tsearch (s->fromname, &printlist, (__compar_fn_t) strverscmp);
658 }
659}
660
661static void
662do_print_human (const void *nodep, VISIT value, int level)
663{
664 if (value == leaf || value == postorder)
665 {
666 const char *s = *(const char **) nodep;
667 size_t len = strlen (s);
668 size_t cnt;
669
670 while (len > 0 && s[len - 1] == '/')
671 --len;
672
673 for (cnt = 0; cnt < len; ++cnt)
674 if (isalnum (s[cnt]))
675 break;
676 if (cnt == len)
677 return;
678
679 if (not_first)
680 {
681 putchar (',');
682 ++column;
683
684 if (column > 2 && column + len > 77)
685 {
686 fputs ("\n ", stdout);
687 column = 2;
688 }
689 else
690 {
691 putchar (' ');
692 ++column;
693 }
694 }
695 else
696 not_first = 1;
697
698 fwrite (s, len, 1, stdout);
699 column += len;
700 }
701}
702
703static void
704do_print (const void *nodep, VISIT value, int level)
705{
706 if (value == leaf || value == postorder)
707 {
708 const char *s = *(const char **) nodep;
709
710 puts (s);
711 }
712}
713
714static void
715internal_function
716add_known_names (struct gconv_module *node)
717{
718 if (node->left != NULL)
719 add_known_names (node->left);
720 if (node->right != NULL)
721 add_known_names (node->right);
722 do
723 {
724 if (strcmp (node->from_string, "INTERNAL") != 0)
725 tsearch (node->from_string, &printlist, (__compar_fn_t) strverscmp);
726 if (strcmp (node->to_string, "INTERNAL") != 0)
727 tsearch (node->to_string, &printlist, (__compar_fn_t) strverscmp);
728
729 node = node->same;
730 }
731 while (node != NULL);
732}
733
734
735static void
736insert_cache (void)
737{
738 const struct gconvcache_header *header;
739 const char *strtab;
740 const struct hash_entry *hashtab;
741 size_t cnt;
742
743 header = (const struct gconvcache_header *) __gconv_get_cache ();
744 strtab = (char *) header + header->string_offset;
745 hashtab = (struct hash_entry *) ((char *) header + header->hash_offset);
746
747 for (cnt = 0; cnt < header->hash_size; ++cnt)
748 if (hashtab[cnt].string_offset != 0)
749 {
750 const char *str = strtab + hashtab[cnt].string_offset;
751
752 if (strcmp (str, "INTERNAL") != 0)
753 tsearch (str, &printlist, (__compar_fn_t) strverscmp);
754 }
755}
756
757
758static void
759internal_function
760print_known_names (void)
761{
762 iconv_t h;
763 void *cache;
764
765 /* We must initialize the internal databases first. */
766 h = iconv_open ("L1", "L1");
767 iconv_close (h);
768
769 /* See whether we have a cache. */
770 cache = __gconv_get_cache ();
771 if (cache != NULL)
772 /* Yep, use only this information. */
773 insert_cache ();
774 else
775 {
776 struct gconv_module *modules;
777
778 /* No, then use the information read from the gconv-modules file.
779 First add the aliases. */
780 twalk (__gconv_get_alias_db (), insert_print_list);
781
782 /* Add the from- and to-names from the known modules. */
783 modules = __gconv_get_modules_db ();
784 if (modules != NULL)
785 add_known_names (modules);
786 }
787
788 bool human_readable = isatty (fileno (stdout));
789
790 if (human_readable)
791 fputs (_("\
792The following list contains all the coded character sets known. This does\n\
793not necessarily mean that all combinations of these names can be used for\n\
794the FROM and TO command line parameters. One coded character set can be\n\
795listed with several different names (aliases).\n\n "), stdout);
796
797 /* Now print the collected names. */
798 column = 2;
799 twalk (printlist, human_readable ? do_print_human : do_print);
800
801 if (human_readable && column != 0)
802 puts ("");
803}
804