SphinxBase 5prealpha
sphinx_lm_convert.c
Go to the documentation of this file.
1/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/* ====================================================================
3 * Copyright (c) 2009 Carnegie Mellon University. All rights
4 * reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * This work was supported in part by funding from the Defense Advanced
19 * Research Projects Agency and the National Science Foundation of the
20 * United States of America, and the CMU Sphinx Speech Consortium.
21 *
22 * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
23 * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
26 * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 *
34 * ====================================================================
35 *
36 */
41#include <sphinxbase/logmath.h>
42#include <sphinxbase/ngram_model.h>
43#include <sphinxbase/cmd_ln.h>
44#include <sphinxbase/ckd_alloc.h>
45#include <sphinxbase/err.h>
46#include <sphinxbase/pio.h>
47#include <sphinxbase/strfuncs.h>
48
49#include <stdio.h>
50#include <string.h>
51#include <math.h>
52
53static const arg_t defn[] = {
54 { "-help",
55 ARG_BOOLEAN,
56 "no",
57 "Shows the usage of the tool"},
58
59 { "-logbase",
60 ARG_FLOAT64,
61 "1.0001",
62 "Base in which all log-likelihoods calculated" },
63
64 { "-i",
65 REQARG_STRING,
66 NULL,
67 "Input language model file (required)"},
68
69 { "-o",
70 REQARG_STRING,
71 NULL,
72 "Output language model file (required)"},
73
74 { "-ifmt",
75 ARG_STRING,
76 NULL,
77 "Input language model format (will guess if not specified)"},
78
79 { "-ofmt",
80 ARG_STRING,
81 NULL,
82 "Output language model file (will guess if not specified)"},
83
84 { "-case",
85 ARG_STRING,
86 NULL,
87 "Ether 'lower' or 'upper' - case fold to lower/upper case (NOT UNICODE AWARE)" },
88
89 { "-mmap",
90 ARG_BOOLEAN,
91 "no",
92 "Use memory-mapped I/O for reading binary LM files"},
93
94 { NULL, 0, NULL, NULL }
95};
96
97static void
98usagemsg(char *pgm)
99{
100 E_INFO("Usage: %s -i <input.lm> \\\n", pgm);
101 E_INFOCONT("\t[-ifmt txt] [-ofmt dmp]\n");
102 E_INFOCONT("\t-o <output.lm.DMP>\n");
103
104 exit(0);
105}
106
107
108int
109main(int argc, char *argv[])
110{
111 cmd_ln_t *config;
112 ngram_model_t *lm = NULL;
113 logmath_t *lmath;
114 int itype, otype;
115 char const *kase;
116
117 if ((config = cmd_ln_parse_r(NULL, defn, argc, argv, TRUE)) == NULL)
118 return 1;
119
120 if (cmd_ln_boolean_r(config, "-help")) {
121 usagemsg(argv[0]);
122 }
123
124 /* Create log math object. */
125 if ((lmath = logmath_init
126 (cmd_ln_float64_r(config, "-logbase"), 0, 0)) == NULL) {
127 E_FATAL("Failed to initialize log math\n");
128 }
129
130 if (cmd_ln_str_r(config, "-i") == NULL || cmd_ln_str_r(config, "-i") == NULL) {
131 E_ERROR("Please specify both input and output models\n");
132 goto error_out;
133 }
134
135 /* Load the input language model. */
136 if (cmd_ln_str_r(config, "-ifmt")) {
137 if ((itype = ngram_str_to_type(cmd_ln_str_r(config, "-ifmt")))
138 == NGRAM_INVALID) {
139 E_ERROR("Invalid input type %s\n", cmd_ln_str_r(config, "-ifmt"));
140 goto error_out;
141 }
142 lm = ngram_model_read(config, cmd_ln_str_r(config, "-i"),
143 itype, lmath);
144 }
145 else {
146 lm = ngram_model_read(config, cmd_ln_str_r(config, "-i"),
147 NGRAM_AUTO, lmath);
148 }
149
150 if (lm == NULL) {
151 E_ERROR("Failed to read the model from the file '%s'\n", cmd_ln_str_r(config, "-i"));
152 goto error_out;
153 }
154
155 /* Guess or set the output language model type. */
156 if (cmd_ln_str_r(config, "-ofmt")) {
157 if ((otype = ngram_str_to_type(cmd_ln_str_r(config, "-ofmt")))
158 == NGRAM_INVALID) {
159 E_ERROR("Invalid output type %s\n", cmd_ln_str_r(config, "-ofmt"));
160 goto error_out;
161 }
162 }
163 else {
164 otype = ngram_file_name_to_type(cmd_ln_str_r(config, "-o"));
165 }
166
167 /* Case fold if requested. */
168 if ((kase = cmd_ln_str_r(config, "-case"))) {
169 if (0 == strcmp(kase, "lower")) {
170 ngram_model_casefold(lm, NGRAM_LOWER);
171 }
172 else if (0 == strcmp(kase, "upper")) {
173 ngram_model_casefold(lm, NGRAM_UPPER);
174 }
175 else {
176 E_ERROR("Unknown value for -case: %s\n", kase);
177 goto error_out;
178 }
179 }
180
181 /* Write the output language model. */
182 if (ngram_model_write(lm, cmd_ln_str_r(config, "-o"), otype) != 0) {
183 E_ERROR("Failed to write language model in format %s to %s\n",
184 ngram_type_to_str(otype), cmd_ln_str_r(config, "-o"));
185 goto error_out;
186 }
187
188 /* That's all folks! */
189 ngram_model_free(lm);
190 if (lmath) {
191 logmath_free(lmath);
192 }
193 if (config) {
194 cmd_ln_free_r(config);
195 }
196 return 0;
197
198error_out:
199 ngram_model_free(lm);
200 if (lmath) {
201 logmath_free(lmath);
202 }
203 if (config) {
204 cmd_ln_free_r(config);
205 }
206 return 1;
207}
ckd_alloc.h
Sphinx's memory allocation/deallocation routines.
cmd_ln.h
Command-line and other configurationparsing and handling.
cmd_ln_boolean_r
#define cmd_ln_boolean_r(c, n)
Retrieve a boolean value from a command-line object.
Definition: cmd_ln.h:334
cmd_ln_free_r
SPHINXBASE_EXPORT int cmd_ln_free_r(cmd_ln_t *cmdln)
Release a command-line argument set and all associated strings.
Definition: cmd_ln.c:1046
ARG_STRING
#define ARG_STRING
String argument (optional).
Definition: cmd_ln.h:114
cmd_ln_str_r
SPHINXBASE_EXPORT char const * cmd_ln_str_r(cmd_ln_t *cmdln, char const *name)
Retrieve a string from a command-line object.
Definition: cmd_ln.c:949
REQARG_STRING
#define REQARG_STRING
Required string argument.
Definition: cmd_ln.h:135
ARG_BOOLEAN
#define ARG_BOOLEAN
Boolean (true/false) argument (optional).
Definition: cmd_ln.h:118
ARG_FLOAT64
#define ARG_FLOAT64
Definition: cmd_ln.h:152
cmd_ln_parse_r
SPHINXBASE_EXPORT cmd_ln_t * cmd_ln_parse_r(cmd_ln_t *inout_cmdln, arg_t const *defn, int32 argc, char *argv[], int32 strict)
Parse a list of strings into argumetns.
Definition: cmd_ln.c:556
err.h
Implementation of logging routines.
E_ERROR
#define E_ERROR(...)
Print error message to error log.
Definition: err.h:104
E_INFO
#define E_INFO(...)
Print logging information to standard error stream.
Definition: err.h:114
E_INFOCONT
#define E_INFOCONT(...)
Continue printing the information to standard error stream.
Definition: err.h:119
E_FATAL
#define E_FATAL(...)
Exit with non-zero status after error message.
Definition: err.h:81
logmath.h
Fast integer logarithmic addition operations.
logmath_free
SPHINXBASE_EXPORT int logmath_free(logmath_t *lmath)
Free a log table.
Definition: logmath.c:342
logmath_init
SPHINXBASE_EXPORT logmath_t * logmath_init(float64 base, int shift, int use_table)
Initialize a log math computation table.
Definition: logmath.c:62
ngram_model.h
N-Gram language models.
NGRAM_INVALID
@ NGRAM_INVALID
Not a valid file type.
Definition: ngram_model.h:77
NGRAM_AUTO
@ NGRAM_AUTO
Determine file type automatically.
Definition: ngram_model.h:78
ngram_model_casefold
SPHINXBASE_EXPORT int ngram_model_casefold(ngram_model_t *model, int kase)
Case-fold word strings in an N-Gram model.
Definition: ngram_model.c:308
ngram_model_read
SPHINXBASE_EXPORT ngram_model_t * ngram_model_read(cmd_ln_t *config, const char *file_name, ngram_file_type_t file_type, logmath_t *lmath)
Read an N-Gram model from a file on disk.
Definition: ngram_model.c:124
ngram_type_to_str
SPHINXBASE_EXPORT char const * ngram_type_to_str(int type)
Get the canonical name for an N-Gram file type.
Definition: ngram_model.c:110
ngram_model_write
SPHINXBASE_EXPORT int ngram_model_write(ngram_model_t *model, const char *file_name, ngram_file_type_t format)
Write an N-Gram model to disk.
Definition: ngram_model.c:178
ngram_file_name_to_type
SPHINXBASE_EXPORT ngram_file_type_t ngram_file_name_to_type(const char *file_name)
Guess the file type for an N-Gram model from the filename.
Definition: ngram_model.c:63
ngram_model_free
SPHINXBASE_EXPORT int ngram_model_free(ngram_model_t *model)
Release memory associated with an N-Gram model.
Definition: ngram_model.c:263
ngram_str_to_type
SPHINXBASE_EXPORT ngram_file_type_t ngram_str_to_type(const char *str_name)
Get the N-Gram file type from a string.
Definition: ngram_model.c:99
pio.h
file IO related operations.
strfuncs.h
Miscellaneous useful string functions.
arg_t
Argument definition structure.
cmd_ln_t
Opaque structure used to hold the results of command-line parsing.
logmath_s
Definition: logmath.c:49
ngram_model_s
Common implementation of ngram_model_t.
Definition: ngram_model_internal.h:55