1/* Convert text in given files from the specified from-set to the to-set.
2 Copyright (C) 1998-2018 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published
8 by the Free Software Foundation; version 2 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <http://www.gnu.org/licenses/>. */
18
19#include <argp.h>
20#include <assert.h>
21#include <ctype.h>
22#include <errno.h>
23#include <error.h>
24#include <fcntl.h>
25#include <iconv.h>
26#include <langinfo.h>
27#include <locale.h>
28#include <search.h>
29#include <stdbool.h>
30#include <stdio.h>
31#include <stdlib.h>
32#include <string.h>
33#include <unistd.h>
34#include <libintl.h>
35#ifdef _POSIX_MAPPED_FILES
36# include <sys/mman.h>
37#endif
38#include <charmap.h>
39#include <gconv_int.h>
40#include "iconv_prog.h"
41#include "iconvconfig.h"
42
43/* Get libc version number. */
44#include "../version.h"
45
46#define PACKAGE _libc_intl_domainname
47
48
49/* Name and version of program. */
50static void print_version (FILE *stream, struct argp_state *state);
51void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
52
53#define OPT_VERBOSE 1000
54#define OPT_LIST 'l'
55
56/* Definitions of arguments for argp functions. */
57static const struct argp_option options[] =
58{
59 { NULL, 0, NULL, 0, N_("Input/Output format specification:") },
60 { "from-code", 'f', N_("NAME"), 0, N_("encoding of original text") },
61 { "to-code", 't', N_("NAME"), 0, N_("encoding for output") },
62 { NULL, 0, NULL, 0, N_("Information:") },
63 { "list", 'l', NULL, 0, N_("list all known coded character sets") },
64 { NULL, 0, NULL, 0, N_("Output control:") },
65 { NULL, 'c', NULL, 0, N_("omit invalid characters from output") },
66 { "output", 'o', N_("FILE"), 0, N_("output file") },
67 { "silent", 's', NULL, 0, N_("suppress warnings") },
68 { "verbose", OPT_VERBOSE, NULL, 0, N_("print progress information") },
69 { NULL, 0, NULL, 0, NULL }
70};
71
72/* Short description of program. */
73static const char doc[] = N_("\
74Convert encoding of given files from one encoding to another.");
75
76/* Strings for arguments in help texts. */
77static const char args_doc[] = N_("[FILE...]");
78
79/* Prototype for option handler. */
80static error_t parse_opt (int key, char *arg, struct argp_state *state);
81
82/* Function to print some extra text in the help message. */
83static char *more_help (int key, const char *text, void *input);
84
85/* Data structure to communicate with argp functions. */
86static struct argp argp =
87{
88 options, parse_opt, args_doc, doc, NULL, more_help
89};
90
91/* Code sets to convert from and to respectively. An empty string as the
92 default causes the 'iconv_open' function to look up the charset of the
93 currently selected locale and use it. */
94static const char *from_code = "";
95static const char *to_code = "";
96
97/* File to write output to. If NULL write to stdout. */
98static const char *output_file;
99
100/* Nonzero if list of all coded character sets is wanted. */
101static int list;
102
103/* If nonzero omit invalid character from output. */
104int omit_invalid;
105
106/* Prototypes for the functions doing the actual work. */
107static int process_block (iconv_t cd, char *addr, size_t len, FILE **output,
108 const char *output_file);
109static int process_fd (iconv_t cd, int fd, FILE **output,
110 const char *output_file);
111static int process_file (iconv_t cd, FILE *input, FILE **output,
112 const char *output_file);
113static void print_known_names (void);
114
115
116int
117main (int argc, char *argv[])
118{
119 int status = EXIT_SUCCESS;
120 int remaining;
121 iconv_t cd;
122 const char *orig_to_code;
123 struct charmap_t *from_charmap = NULL;
124 struct charmap_t *to_charmap = NULL;
125
126 /* Set locale via LC_ALL. */
127 setlocale (LC_ALL, "");
128
129 /* Set the text message domain. */
130 textdomain (_libc_intl_domainname);
131
132 /* Parse and process arguments. */
133 argp_parse (&argp, argc, argv, 0, &remaining, NULL);
134
135 /* List all coded character sets if wanted. */
136 if (list)
137 {
138 print_known_names ();
139 exit (EXIT_SUCCESS);
140 }
141
142 /* If we have to ignore errors make sure we use the appropriate name for
143 the to-character-set. */
144 orig_to_code = to_code;
145 if (omit_invalid)
146 {
147 const char *errhand = strchrnul (to_code, '/');
148 int nslash = 2;
149 char *newp;
150 char *cp;
151
152 if (*errhand == '/')
153 {
154 --nslash;
155 errhand = strchrnul (errhand + 1, '/');
156
157 if (*errhand == '/')
158 {
159 --nslash;
160 errhand = strchr (errhand, '\0');
161 }
162 }
163
164 newp = (char *) alloca (errhand - to_code + nslash + 7 + 1);
165 cp = mempcpy (newp, to_code, errhand - to_code);
166 while (nslash-- > 0)
167 *cp++ = '/';
168 if (cp[-1] != '/')
169 *cp++ = ',';
170 memcpy (cp, "IGNORE", sizeof ("IGNORE"));
171
172 to_code = newp;
173 }
174
175 /* POSIX 1003.2b introduces a silly thing: the arguments to -t anf -f
176 can be file names of charmaps. In this case iconv will have to read
177 those charmaps and use them to do the conversion. But there are
178 holes in the specification. There is nothing said that if -f is a
179 charmap filename that -t must be, too. And vice versa. There is
180 also no word about the symbolic names used. What if they don't
181 match? */
182 if (strchr (from_code, '/') != NULL)
183 /* The from-name might be a charmap file name. Try reading the
184 file. */
185 from_charmap = charmap_read (from_code, /*0, 1*/1, 0, 0, 0);
186
187 if (strchr (orig_to_code, '/') != NULL)
188 /* The to-name might be a charmap file name. Try reading the
189 file. */
190 to_charmap = charmap_read (orig_to_code, /*0, 1,*/1, 0, 0, 0);
191
192
193 /* At this point we have to handle two cases. The first one is
194 where a charmap is used for the from- or to-charset, or both. We
195 handle this special since it is very different from the sane way of
196 doing things. The other case allows converting using the iconv()
197 function. */
198 if (from_charmap != NULL || to_charmap != NULL)
199 /* Construct the conversion table and do the conversion. */
200 status = charmap_conversion (from_code, from_charmap, to_code, to_charmap,
201 argc, remaining, argv, output_file);
202 else
203 {
204 /* Let's see whether we have these coded character sets. */
205 cd = iconv_open (to_code, from_code);
206 if (cd == (iconv_t) -1)
207 {
208 if (errno == EINVAL)
209 {
210 /* Try to be nice with the user and tell her which of the
211 two encoding names is wrong. This is possible because
212 all supported encodings can be converted from/to Unicode,
213 in other words, because the graph of encodings is
214 connected. */
215 bool from_wrong =
216 (iconv_open ("UTF-8", from_code) == (iconv_t) -1
217 && errno == EINVAL);
218 bool to_wrong =
219 (iconv_open (to_code, "UTF-8") == (iconv_t) -1
220 && errno == EINVAL);
221 const char *from_pretty =
222 (from_code[0] ? from_code : nl_langinfo (CODESET));
223 const char *to_pretty =
224 (orig_to_code[0] ? orig_to_code : nl_langinfo (CODESET));
225
226 if (from_wrong)
227 {
228 if (to_wrong)
229 error (0, 0,
230 _("\
231conversions from `%s' and to `%s' are not supported"),
232 from_pretty, to_pretty);
233 else
234 error (0, 0,
235 _("conversion from `%s' is not supported"),
236 from_pretty);
237 }
238 else
239 {
240 if (to_wrong)
241 error (0, 0,
242 _("conversion to `%s' is not supported"),
243 to_pretty);
244 else
245 error (0, 0,
246 _("conversion from `%s' to `%s' is not supported"),
247 from_pretty, to_pretty);
248 }
249
250 argp_help (&argp, stderr, ARGP_HELP_SEE,
251 program_invocation_short_name);
252 exit (1);
253 }
254 else
255 error (EXIT_FAILURE, errno,
256 _("failed to start conversion processing"));
257 }
258
259 /* The output file. Will be opened when we are ready to produce
260 output. */
261 FILE *output = NULL;
262
263 /* Now process the remaining files. Write them to stdout or the file
264 specified with the `-o' parameter. If we have no file given as
265 the parameter process all from stdin. */
266 if (remaining == argc)
267 {
268 if (process_file (cd, stdin, &output, output_file) != 0)
269 status = EXIT_FAILURE;
270 }
271 else
272 do
273 {
274#ifdef _POSIX_MAPPED_FILES
275 struct stat64 st;
276 char *addr;
277#endif
278 int fd, ret;
279
280 if (verbose)
281 fprintf (stderr, "%s:\n", argv[remaining]);
282 if (strcmp (argv[remaining], "-") == 0)
283 fd = 0;
284 else
285 {
286 fd = open (argv[remaining], O_RDONLY);
287
288 if (fd == -1)
289 {
290 error (0, errno, _("cannot open input file `%s'"),
291 argv[remaining]);
292 status = EXIT_FAILURE;
293 continue;
294 }
295 }
296
297#ifdef _POSIX_MAPPED_FILES
298 /* We have possibilities for reading the input file. First try
299 to mmap() it since this will provide the fastest solution. */
300 if (fstat64 (fd, &st) == 0
301 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
302 fd, 0)) != MAP_FAILED))
303 {
304 /* Yes, we can use mmap(). The descriptor is not needed
305 anymore. */
306 if (close (fd) != 0)
307 error (EXIT_FAILURE, errno,
308 _("error while closing input `%s'"),
309 argv[remaining]);
310
311 ret = process_block (cd, addr, st.st_size, &output,
312 output_file);
313
314 /* We don't need the input data anymore. */
315 munmap ((void *) addr, st.st_size);
316
317 if (ret != 0)
318 {
319 status = EXIT_FAILURE;
320
321 if (ret < 0)
322 /* We cannot go on with producing output since it might
323 lead to problem because the last output might leave
324 the output stream in an undefined state. */
325 break;
326 }
327 }
328 else
329#endif /* _POSIX_MAPPED_FILES */
330 {
331 /* Read the file in pieces. */
332 ret = process_fd (cd, fd, &output, output_file);
333
334 /* Now close the file. */
335 close (fd);
336
337 if (ret != 0)
338 {
339 /* Something went wrong. */
340 status = EXIT_FAILURE;
341
342 if (ret < 0)
343 /* We cannot go on with producing output since it might
344 lead to problem because the last output might leave
345 the output stream in an undefined state. */
346 break;
347 }
348 }
349 }
350 while (++remaining < argc);
351
352 /* Close the output file now. */
353 if (output != NULL && fclose (output))
354 error (EXIT_FAILURE, errno, _("error while closing output file"));
355 }
356
357 return status;
358}
359
360
361/* Handle program arguments. */
362static error_t
363parse_opt (int key, char *arg, struct argp_state *state)
364{
365 switch (key)
366 {
367 case 'f':
368 from_code = arg;
369 break;
370 case 't':
371 to_code = arg;
372 break;
373 case 'o':
374 output_file = arg;
375 break;
376 case 's':
377 /* Nothing, for now at least. We are not giving out any information
378 about missing character or so. */
379 break;
380 case 'c':
381 /* Omit invalid characters from output. */
382 omit_invalid = 1;
383 break;
384 case OPT_VERBOSE:
385 verbose = 1;
386 break;
387 case OPT_LIST:
388 list = 1;
389 break;
390 default:
391 return ARGP_ERR_UNKNOWN;
392 }
393 return 0;
394}
395
396
397static char *
398more_help (int key, const char *text, void *input)
399{
400 char *tp = NULL;
401 switch (key)
402 {
403 case ARGP_KEY_HELP_EXTRA:
404 /* We print some extra information. */
405 if (asprintf (&tp, gettext ("\
406For bug reporting instructions, please see:\n\
407%s.\n"), REPORT_BUGS_TO) < 0)
408 return NULL;
409 return tp;
410 default:
411 break;
412 }
413 return (char *) text;
414}
415
416
417/* Print the version information. */
418static void
419print_version (FILE *stream, struct argp_state *state)
420{
421 fprintf (stream, "iconv %s%s\n", PKGVERSION, VERSION);
422 fprintf (stream, gettext ("\
423Copyright (C) %s Free Software Foundation, Inc.\n\
424This is free software; see the source for copying conditions. There is NO\n\
425warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n\
426"), "2018");
427 fprintf (stream, gettext ("Written by %s.\n"), "Ulrich Drepper");
428}
429
430
431static int
432write_output (const char *outbuf, const char *outptr, FILE **output,
433 const char *output_file)
434{
435 /* We have something to write out. */
436 int errno_save = errno;
437
438 if (*output == NULL)
439 {
440 /* Determine output file. */
441 if (output_file != NULL && strcmp (output_file, "-") != 0)
442 {
443 *output = fopen (output_file, "w");
444 if (*output == NULL)
445 error (EXIT_FAILURE, errno, _("cannot open output file"));
446 }
447 else
448 *output = stdout;
449 }
450
451 if (fwrite (outbuf, 1, outptr - outbuf, *output) < (size_t) (outptr - outbuf)
452 || ferror (*output))
453 {
454 /* Error occurred while printing the result. */
455 error (0, 0, _("\
456conversion stopped due to problem in writing the output"));
457 return -1;
458 }
459
460 errno = errno_save;
461
462 return 0;
463}
464
465
466static int
467process_block (iconv_t cd, char *addr, size_t len, FILE **output,
468 const char *output_file)
469{
470#define OUTBUF_SIZE 32768
471 const char *start = addr;
472 char outbuf[OUTBUF_SIZE];
473 char *outptr;
474 size_t outlen;
475 size_t n;
476 int ret = 0;
477
478 while (len > 0)
479 {
480 outptr = outbuf;
481 outlen = OUTBUF_SIZE;
482 n = iconv (cd, &addr, &len, &outptr, &outlen);
483
484 if (n == (size_t) -1 && omit_invalid && errno == EILSEQ)
485 {
486 ret = 1;
487 if (len == 0)
488 n = 0;
489 else
490 errno = E2BIG;
491 }
492
493 if (outptr != outbuf)
494 {
495 ret = write_output (outbuf, outptr, output, output_file);
496 if (ret != 0)
497 break;
498 }
499
500 if (n != (size_t) -1)
501 {
502 /* All the input test is processed. For state-dependent
503 character sets we have to flush the state now. */
504 outptr = outbuf;
505 outlen = OUTBUF_SIZE;
506 n = iconv (cd, NULL, NULL, &outptr, &outlen);
507
508 if (outptr != outbuf)
509 {
510 ret = write_output (outbuf, outptr, output, output_file);
511 if (ret != 0)
512 break;
513 }
514
515 if (n != (size_t) -1)
516 break;
517
518 if (omit_invalid && errno == EILSEQ)
519 {
520 ret = 1;
521 break;
522 }
523 }
524
525 if (errno != E2BIG)
526 {
527 /* iconv() ran into a problem. */
528 switch (errno)
529 {
530 case EILSEQ:
531 if (! omit_invalid)
532 error (0, 0, _("illegal input sequence at position %ld"),
533 (long int) (addr - start));
534 break;
535 case EINVAL:
536 error (0, 0, _("\
537incomplete character or shift sequence at end of buffer"));
538 break;
539 case EBADF:
540 error (0, 0, _("internal error (illegal descriptor)"));
541 break;
542 default:
543 error (0, 0, _("unknown iconv() error %d"), errno);
544 break;
545 }
546
547 return -1;
548 }
549 }
550
551 return ret;
552}
553
554
555static int
556process_fd (iconv_t cd, int fd, FILE **output, const char *output_file)
557{
558 /* we have a problem with reading from a desriptor since we must not
559 provide the iconv() function an incomplete character or shift
560 sequence at the end of the buffer. Since we have to deal with
561 arbitrary encodings we must read the whole text in a buffer and
562 process it in one step. */
563 static char *inbuf = NULL;
564 static size_t maxlen = 0;
565 char *inptr = NULL;
566 size_t actlen = 0;
567
568 while (actlen < maxlen)
569 {
570 ssize_t n = read (fd, inptr, maxlen - actlen);
571
572 if (n == 0)
573 /* No more text to read. */
574 break;
575
576 if (n == -1)
577 {
578 /* Error while reading. */
579 error (0, errno, _("error while reading the input"));
580 return -1;
581 }
582
583 inptr += n;
584 actlen += n;
585 }
586
587 if (actlen == maxlen)
588 while (1)
589 {
590 ssize_t n;
591 char *new_inbuf;
592
593 /* Increase the buffer. */
594 new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
595 if (new_inbuf == NULL)
596 {
597 error (0, errno, _("unable to allocate buffer for input"));
598 return -1;
599 }
600 inbuf = new_inbuf;
601 maxlen += 32768;
602 inptr = inbuf + actlen;
603
604 do
605 {
606 n = read (fd, inptr, maxlen - actlen);
607
608 if (n == 0)
609 /* No more text to read. */
610 break;
611
612 if (n == -1)
613 {
614 /* Error while reading. */
615 error (0, errno, _("error while reading the input"));
616 return -1;
617 }
618
619 inptr += n;
620 actlen += n;
621 }
622 while (actlen < maxlen);
623
624 if (n == 0)
625 /* Break again so we leave both loops. */
626 break;
627 }
628
629 /* Now we have all the input in the buffer. Process it in one run. */
630 return process_block (cd, inbuf, actlen, output, output_file);
631}
632
633
634static int
635process_file (iconv_t cd, FILE *input, FILE **output, const char *output_file)
636{
637 /* This should be safe since we use this function only for `stdin' and
638 we haven't read anything so far. */
639 return process_fd (cd, fileno (input), output, output_file);
640}
641
642
643/* Print all known character sets/encodings. */
644static void *printlist;
645static size_t column;
646static int not_first;
647
648static void
649insert_print_list (const void *nodep, VISIT value, int level)
650{
651 if (value == leaf || value == postorder)
652 {
653 const struct gconv_alias *s = *(const struct gconv_alias **) nodep;
654 tsearch (s->fromname, &printlist, (__compar_fn_t) strverscmp);
655 }
656}
657
658static void
659do_print_human (const void *nodep, VISIT value, int level)
660{
661 if (value == leaf || value == postorder)
662 {
663 const char *s = *(const char **) nodep;
664 size_t len = strlen (s);
665 size_t cnt;
666
667 while (len > 0 && s[len - 1] == '/')
668 --len;
669
670 for (cnt = 0; cnt < len; ++cnt)
671 if (isalnum (s[cnt]))
672 break;
673 if (cnt == len)
674 return;
675
676 if (not_first)
677 {
678 putchar (',');
679 ++column;
680
681 if (column > 2 && column + len > 77)
682 {
683 fputs ("\n ", stdout);
684 column = 2;
685 }
686 else
687 {
688 putchar (' ');
689 ++column;
690 }
691 }
692 else
693 not_first = 1;
694
695 fwrite (s, len, 1, stdout);
696 column += len;
697 }
698}
699
700static void
701do_print (const void *nodep, VISIT value, int level)
702{
703 if (value == leaf || value == postorder)
704 {
705 const char *s = *(const char **) nodep;
706
707 puts (s);
708 }
709}
710
711static void
712add_known_names (struct gconv_module *node)
713{
714 if (node->left != NULL)
715 add_known_names (node->left);
716 if (node->right != NULL)
717 add_known_names (node->right);
718 do
719 {
720 if (strcmp (node->from_string, "INTERNAL") != 0)
721 tsearch (node->from_string, &printlist, (__compar_fn_t) strverscmp);
722 if (strcmp (node->to_string, "INTERNAL") != 0)
723 tsearch (node->to_string, &printlist, (__compar_fn_t) strverscmp);
724
725 node = node->same;
726 }
727 while (node != NULL);
728}
729
730
731static void
732insert_cache (void)
733{
734 const struct gconvcache_header *header;
735 const char *strtab;
736 const struct hash_entry *hashtab;
737 size_t cnt;
738
739 header = (const struct gconvcache_header *) __gconv_get_cache ();
740 strtab = (char *) header + header->string_offset;
741 hashtab = (struct hash_entry *) ((char *) header + header->hash_offset);
742
743 for (cnt = 0; cnt < header->hash_size; ++cnt)
744 if (hashtab[cnt].string_offset != 0)
745 {
746 const char *str = strtab + hashtab[cnt].string_offset;
747
748 if (strcmp (str, "INTERNAL") != 0)
749 tsearch (str, &printlist, (__compar_fn_t) strverscmp);
750 }
751}
752
753
754static void
755print_known_names (void)
756{
757 iconv_t h;
758 void *cache;
759
760 /* We must initialize the internal databases first. */
761 h = iconv_open ("L1", "L1");
762 iconv_close (h);
763
764 /* See whether we have a cache. */
765 cache = __gconv_get_cache ();
766 if (cache != NULL)
767 /* Yep, use only this information. */
768 insert_cache ();
769 else
770 {
771 struct gconv_module *modules;
772
773 /* No, then use the information read from the gconv-modules file.
774 First add the aliases. */
775 twalk (__gconv_get_alias_db (), insert_print_list);
776
777 /* Add the from- and to-names from the known modules. */
778 modules = __gconv_get_modules_db ();
779 if (modules != NULL)
780 add_known_names (modules);
781 }
782
783 bool human_readable = isatty (fileno (stdout));
784
785 if (human_readable)
786 fputs (_("\
787The following list contains all the coded character sets known. This does\n\
788not necessarily mean that all combinations of these names can be used for\n\
789the FROM and TO command line parameters. One coded character set can be\n\
790listed with several different names (aliases).\n\n "), stdout);
791
792 /* Now print the collected names. */
793 column = 2;
794 twalk (printlist, human_readable ? do_print_human : do_print);
795
796 if (human_readable && column != 0)
797 puts ("");
798}
799