PocketSphinx 5prealpha
dict.c
1/* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
2/* ====================================================================
3 * Copyright (c) 1999-2004 Carnegie Mellon University. All rights
4 * reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 *
18 * This work was supported in part by funding from the Defense Advanced
19 * Research Projects Agency and the National Science Foundation of the
20 * United States of America, and the CMU Sphinx Speech Consortium.
21 *
22 * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
23 * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
26 * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 *
34 * ====================================================================
35 *
36 */
37
38/* System headers. */
39#include <string.h>
40
41/* SphinxBase headers. */
42#include <sphinxbase/pio.h>
43#include <sphinxbase/strfuncs.h>
44
45/* Local headers. */
46#include "dict.h"
47
48
49#define DELIM " \t\n" /* Set of field separator characters */
50#define DEFAULT_NUM_PHONE (MAX_S3CIPID+1)
51
52#if WIN32
53#define snprintf sprintf_s
54#endif
55
56extern const char *const cmu6_lts_phone_table[];
57
58static s3cipid_t
59dict_ciphone_id(dict_t * d, const char *str)
60{
61 if (d->nocase)
62 return bin_mdef_ciphone_id_nocase(d->mdef, str);
63 else
64 return bin_mdef_ciphone_id(d->mdef, str);
65}
66
67
68const char *
69dict_ciphone_str(dict_t * d, s3wid_t wid, int32 pos)
70{
71 assert(d != NULL);
72 assert((wid >= 0) && (wid < d->n_word));
73 assert((pos >= 0) && (pos < d->word[wid].pronlen));
74
75 return bin_mdef_ciphone_str(d->mdef, d->word[wid].ciphone[pos]);
76}
77
78
79s3wid_t
80dict_add_word(dict_t * d, char const *word, s3cipid_t const * p, int32 np)
81{
82 int32 len;
83 dictword_t *wordp;
84 s3wid_t newwid;
85 char *wword;
86
87 if (d->n_word >= d->max_words) {
88 E_INFO("Reallocating to %d KiB for word entries\n",
89 (d->max_words + S3DICT_INC_SZ) * sizeof(dictword_t) / 1024);
90 d->word =
91 (dictword_t *) ckd_realloc(d->word,
92 (d->max_words +
93 S3DICT_INC_SZ) * sizeof(dictword_t));
94 d->max_words = d->max_words + S3DICT_INC_SZ;
95 }
96
97 wordp = d->word + d->n_word;
98 wordp->word = (char *) ckd_salloc(word); /* Freed in dict_free */
99
100 /* Determine base/alt wids */
101 wword = ckd_salloc(word);
102 if ((len = dict_word2basestr(wword)) > 0) {
103 int32 w;
104
105 /* Truncated to a baseword string; find its ID */
106 if (hash_table_lookup_int32(d->ht, wword, &w) < 0) {
107 E_ERROR("Missing base word for: %s\n", word);
108 ckd_free(wword);
109 ckd_free(wordp->word);
110 wordp->word = NULL;
111 return BAD_S3WID;
112 }
113
114 /* Link into alt list */
115 wordp->basewid = w;
116 wordp->alt = d->word[w].alt;
117 d->word[w].alt = d->n_word;
118 } else {
119 wordp->alt = BAD_S3WID;
120 wordp->basewid = d->n_word;
121 }
122 ckd_free(wword);
123
124 /* Associate word string with d->n_word in hash table */
125 if (hash_table_enter_int32(d->ht, wordp->word, d->n_word) != d->n_word) {
126 ckd_free(wordp->word);
127 wordp->word = NULL;
128 return BAD_S3WID;
129 }
130
131 /* Fill in word entry, and set defaults */
132 if (p && (np > 0)) {
133 wordp->ciphone = (s3cipid_t *) ckd_malloc(np * sizeof(s3cipid_t)); /* Freed in dict_free */
134 memcpy(wordp->ciphone, p, np * sizeof(s3cipid_t));
135 wordp->pronlen = np;
136 }
137 else {
138 wordp->ciphone = NULL;
139 wordp->pronlen = 0;
140 }
141
142 newwid = d->n_word++;
143
144 return newwid;
145}
146
147
148static int32
149dict_read(FILE * fp, dict_t * d)
150{
151 lineiter_t *li;
152 char **wptr;
153 s3cipid_t *p;
154 int32 lineno, nwd;
155 s3wid_t w;
156 int32 i, maxwd;
157 size_t stralloc, phnalloc;
158
159 maxwd = 512;
160 p = (s3cipid_t *) ckd_calloc(maxwd + 4, sizeof(*p));
161 wptr = (char **) ckd_calloc(maxwd, sizeof(char *)); /* Freed below */
162
163 lineno = 0;
164 stralloc = phnalloc = 0;
165 for (li = lineiter_start(fp); li; li = lineiter_next(li)) {
166 lineno++;
167 if (0 == strncmp(li->buf, "##", 2)
168 || 0 == strncmp(li->buf, ";;", 2))
169 continue;
170
171 if ((nwd = str2words(li->buf, wptr, maxwd)) < 0) {
172 /* Increase size of p, wptr. */
173 nwd = str2words(li->buf, NULL, 0);
174 assert(nwd > maxwd); /* why else would it fail? */
175 maxwd = nwd;
176 p = (s3cipid_t *) ckd_realloc(p, (maxwd + 4) * sizeof(*p));
177 wptr = (char **) ckd_realloc(wptr, maxwd * sizeof(*wptr));
178 }
179
180 if (nwd == 0) /* Empty line */
181 continue;
182 /* wptr[0] is the word-string and wptr[1..nwd-1] the pronunciation sequence */
183 if (nwd == 1) {
184 E_ERROR("Line %d: No pronunciation for word '%s'; ignored\n",
185 lineno, wptr[0]);
186 continue;
187 }
188
189
190 /* Convert pronunciation string to CI-phone-ids */
191 for (i = 1; i < nwd; i++) {
192 p[i - 1] = dict_ciphone_id(d, wptr[i]);
193 if (NOT_S3CIPID(p[i - 1])) {
194 E_ERROR("Line %d: Phone '%s' is mising in the acoustic model; word '%s' ignored\n",
195 lineno, wptr[i], wptr[0]);
196 break;
197 }
198 }
199
200 if (i == nwd) { /* All CI-phones successfully converted to IDs */
201 w = dict_add_word(d, wptr[0], p, nwd - 1);
202 if (NOT_S3WID(w))
203 E_ERROR
204 ("Line %d: Failed to add the word '%s' (duplicate?); ignored\n",
205 lineno, wptr[0]);
206 else {
207 stralloc += strlen(d->word[w].word);
208 phnalloc += d->word[w].pronlen * sizeof(s3cipid_t);
209 }
210 }
211 }
212 E_INFO("Dictionary size %d, allocated %d KiB for strings, %d KiB for phones\n",
213 dict_size(d), (int)stralloc / 1024, (int)phnalloc / 1024);
214 ckd_free(p);
215 ckd_free(wptr);
216
217 return 0;
218}
219
220int
221dict_write(dict_t *dict, char const *filename, char const *format)
222{
223 FILE *fh;
224 int i;
225
226 if ((fh = fopen(filename, "w")) == NULL) {
227 E_ERROR_SYSTEM("Failed to open '%s'", filename);
228 return -1;
229 }
230 for (i = 0; i < dict->n_word; ++i) {
231 char *phones;
232 int j, phlen;
233 if (!dict_real_word(dict, i))
234 continue;
235 for (phlen = j = 0; j < dict_pronlen(dict, i); ++j)
236 phlen += strlen(dict_ciphone_str(dict, i, j)) + 1;
237 phones = ckd_calloc(1, phlen);
238 for (j = 0; j < dict_pronlen(dict, i); ++j) {
239 strcat(phones, dict_ciphone_str(dict, i, j));
240 if (j != dict_pronlen(dict, i) - 1)
241 strcat(phones, " ");
242 }
243 fprintf(fh, "%-30s %s\n", dict_wordstr(dict, i), phones);
244 ckd_free(phones);
245 }
246 fclose(fh);
247 return 0;
248}
249
250
251dict_t *
252dict_init(cmd_ln_t *config, bin_mdef_t * mdef)
253{
254 FILE *fp, *fp2;
255 int32 n;
256 lineiter_t *li;
257 dict_t *d;
258 s3cipid_t sil;
259 char const *dictfile = NULL, *fillerfile = NULL;
260
261 if (config) {
262 dictfile = cmd_ln_str_r(config, "-dict");
263 fillerfile = cmd_ln_str_r(config, "_fdict");
264 }
265
266 /*
267 * First obtain #words in dictionary (for hash table allocation).
268 * Reason: The PC NT system doesn't like to grow memory gradually. Better to allocate
269 * all the required memory in one go.
270 */
271 fp = NULL;
272 n = 0;
273 if (dictfile) {
274 if ((fp = fopen(dictfile, "r")) == NULL) {
275 E_ERROR_SYSTEM("Failed to open dictionary file '%s' for reading", dictfile);
276 return NULL;
277 }
278 for (li = lineiter_start(fp); li; li = lineiter_next(li)) {
279 if (0 != strncmp(li->buf, "##", 2)
280 && 0 != strncmp(li->buf, ";;", 2))
281 n++;
282 }
283 fseek(fp, 0L, SEEK_SET);
284 }
285
286 fp2 = NULL;
287 if (fillerfile) {
288 if ((fp2 = fopen(fillerfile, "r")) == NULL) {
289 E_ERROR_SYSTEM("Failed to open filler dictionary file '%s' for reading", fillerfile);
290 fclose(fp);
291 return NULL;
292 }
293 for (li = lineiter_start(fp2); li; li = lineiter_next(li)) {
294 if (0 != strncmp(li->buf, "##", 2)
295 && 0 != strncmp(li->buf, ";;", 2))
296 n++;
297 }
298 fseek(fp2, 0L, SEEK_SET);
299 }
300
301 /*
302 * Allocate dict entries. HACK!! Allow some extra entries for words not in file.
303 * Also check for type size restrictions.
304 */
305 d = (dict_t *) ckd_calloc(1, sizeof(dict_t)); /* freed in dict_free() */
306 d->refcnt = 1;
307 d->max_words =
308 (n + S3DICT_INC_SZ < MAX_S3WID) ? n + S3DICT_INC_SZ : MAX_S3WID;
309 if (n >= MAX_S3WID) {
310 E_ERROR("Number of words in dictionaries (%d) exceeds limit (%d)\n", n,
311 MAX_S3WID);
312 fclose(fp);
313 fclose(fp2);
314 ckd_free(d);
315 return NULL;
316 }
317
318 E_INFO("Allocating %d * %d bytes (%d KiB) for word entries\n",
319 d->max_words, sizeof(dictword_t),
320 d->max_words * sizeof(dictword_t) / 1024);
321 d->word = (dictword_t *) ckd_calloc(d->max_words, sizeof(dictword_t)); /* freed in dict_free() */
322 d->n_word = 0;
323 if (mdef)
324 d->mdef = bin_mdef_retain(mdef);
325
326 /* Create new hash table for word strings; case-insensitive word strings */
327 if (config && cmd_ln_exists_r(config, "-dictcase"))
328 d->nocase = cmd_ln_boolean_r(config, "-dictcase");
329 d->ht = hash_table_new(d->max_words, d->nocase);
330
331 /* Digest main dictionary file */
332 if (fp) {
333 E_INFO("Reading main dictionary: %s\n", dictfile);
334 dict_read(fp, d);
335 fclose(fp);
336 E_INFO("%d words read\n", d->n_word);
337 }
338
339 if (dict_wordid(d, S3_START_WORD) != BAD_S3WID) {
340 E_ERROR("Remove sentence start word '<s>' from the dictionary\n");
341 dict_free(d);
342 return NULL;
343 }
344 if (dict_wordid(d, S3_FINISH_WORD) != BAD_S3WID) {
345 E_ERROR("Remove sentence start word '</s>' from the dictionary\n");
346 dict_free(d);
347 return NULL;
348 }
349 if (dict_wordid(d, S3_SILENCE_WORD) != BAD_S3WID) {
350 E_ERROR("Remove silence word '<sil>' from the dictionary\n");
351 dict_free(d);
352 return NULL;
353 }
354
355 /* Now the filler dictionary file, if it exists */
356 d->filler_start = d->n_word;
357 if (fillerfile) {
358 E_INFO("Reading filler dictionary: %s\n", fillerfile);
359 dict_read(fp2, d);
360 fclose(fp2);
361 E_INFO("%d words read\n", d->n_word - d->filler_start);
362 }
363 if (mdef)
364 sil = bin_mdef_silphone(mdef);
365 else
366 sil = 0;
367 if (dict_wordid(d, S3_START_WORD) == BAD_S3WID) {
368 dict_add_word(d, S3_START_WORD, &sil, 1);
369 }
370 if (dict_wordid(d, S3_FINISH_WORD) == BAD_S3WID) {
371 dict_add_word(d, S3_FINISH_WORD, &sil, 1);
372 }
373 if (dict_wordid(d, S3_SILENCE_WORD) == BAD_S3WID) {
374 dict_add_word(d, S3_SILENCE_WORD, &sil, 1);
375 }
376
377 d->filler_end = d->n_word - 1;
378
379 /* Initialize distinguished word-ids */
380 d->startwid = dict_wordid(d, S3_START_WORD);
381 d->finishwid = dict_wordid(d, S3_FINISH_WORD);
382 d->silwid = dict_wordid(d, S3_SILENCE_WORD);
383
384 if ((d->filler_start > d->filler_end)
385 || (!dict_filler_word(d, d->silwid))) {
386 E_ERROR("Word '%s' must occur (only) in filler dictionary\n",
387 S3_SILENCE_WORD);
388 dict_free(d);
389 return NULL;
390 }
391
392 /* No check that alternative pronunciations for filler words are in filler range!! */
393
394 return d;
395}
396
397
398s3wid_t
399dict_wordid(dict_t *d, const char *word)
400{
401 int32 w;
402
403 assert(d);
404 assert(word);
405
406 if (hash_table_lookup_int32(d->ht, word, &w) < 0)
407 return (BAD_S3WID);
408 return w;
409}
410
411
412int
413dict_filler_word(dict_t *d, s3wid_t w)
414{
415 assert(d);
416 assert((w >= 0) && (w < d->n_word));
417
418 w = dict_basewid(d, w);
419 if ((w == d->startwid) || (w == d->finishwid))
420 return 0;
421 if ((w >= d->filler_start) && (w <= d->filler_end))
422 return 1;
423 return 0;
424}
425
426int
427dict_real_word(dict_t *d, s3wid_t w)
428{
429 assert(d);
430 assert((w >= 0) && (w < d->n_word));
431
432 w = dict_basewid(d, w);
433 if ((w == d->startwid) || (w == d->finishwid))
434 return 0;
435 if ((w >= d->filler_start) && (w <= d->filler_end))
436 return 0;
437 return 1;
438}
439
440
441int32
442dict_word2basestr(char *word)
443{
444 int32 i, len;
445
446 len = strlen(word);
447 if (word[len - 1] == ')') {
448 for (i = len - 2; (i > 0) && (word[i] != '('); --i);
449
450 if (i > 0) {
451 /* The word is of the form <baseword>(...); strip from left-paren */
452 word[i] = '\0';
453 return i;
454 }
455 }
456
457 return -1;
458}
459
460dict_t *
461dict_retain(dict_t *d)
462{
463 ++d->refcnt;
464 return d;
465}
466
467int
468dict_free(dict_t * d)
469{
470 int i;
471 dictword_t *word;
472
473 if (d == NULL)
474 return 0;
475 if (--d->refcnt > 0)
476 return d->refcnt;
477
478 /* First Step, free all memory allocated for each word */
479 for (i = 0; i < d->n_word; i++) {
480 word = (dictword_t *) & (d->word[i]);
481 if (word->word)
482 ckd_free((void *) word->word);
483 if (word->ciphone)
484 ckd_free((void *) word->ciphone);
485 }
486
487 if (d->word)
488 ckd_free((void *) d->word);
489 if (d->ht)
490 hash_table_free(d->ht);
491 if (d->mdef)
492 bin_mdef_free(d->mdef);
493 ckd_free((void *) d);
494
495 return 0;
496}
497
498void
499dict_report(dict_t * d)
500{
501 E_INFO_NOFN("Initialization of dict_t, report:\n");
502 E_INFO_NOFN("Max word: %d\n", d->max_words);
503 E_INFO_NOFN("No of word: %d\n", d->n_word);
504 E_INFO_NOFN("\n");
505}
Operations on dictionary.
#define dict_size(d)
Packaged macro access to dictionary members.
Definition: dict.h:151
#define BAD_S3WID
Dictionary word id.
Definition: s3types.h:90
int16 s3cipid_t
Size definitions for more semantially meaningful units.
Definition: s3types.h:63
a structure for a dictionary.
Definition: dict.h:76
s3wid_t startwid
FOR INTERNAL-USE ONLY.
Definition: dict.h:85
s3wid_t finishwid
FOR INTERNAL-USE ONLY.
Definition: dict.h:86
bin_mdef_t * mdef
Model definition used for phone IDs; NULL if none used.
Definition: dict.h:78
hash_table_t * ht
Hash table for mapping word strings to word ids.
Definition: dict.h:80
int32 filler_end
Last filler word id (read from filler dict)
Definition: dict.h:84
dictword_t * word
Array of entries in dictionary.
Definition: dict.h:79
s3wid_t silwid
FOR INTERNAL-USE ONLY.
Definition: dict.h:87
int32 filler_start
First filler word id (read from filler dict)
Definition: dict.h:83
int32 n_word
#Occupied entries in dict; ie, excluding empty slots
Definition: dict.h:82
int32 max_words
#Entries allocated in dict, including empty slots
Definition: dict.h:81
a structure for one dictionary word.
Definition: dict.h:63
char * word
Ascii word string.
Definition: dict.h:64
int32 pronlen
Pronunciation length.
Definition: dict.h:66
s3wid_t basewid
Base pronunciation id.
Definition: dict.h:68
s3wid_t alt
Next alternative pronunciation id, NOT_S3WID if none.
Definition: dict.h:67
s3cipid_t * ciphone
Pronunciation.
Definition: dict.h:65