This file is indexed.

/usr/include/speech_tools/EST_Token.h is in libestools2.1-dev 1:2.1~release-2build2.

This file is owned by root:root, with mode 0o644.

The actual contents of the file can be viewed below.

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
/*************************************************************************/
/*                                                                       */
/*                Centre for Speech Technology Research                  */
/*                     University of Edinburgh, UK                       */
/*                         Copyright (c) 1996                            */
/*                        All Rights Reserved.                           */
/*                                                                       */
/*  Permission is hereby granted, free of charge, to use and distribute  */
/*  this software and its documentation without restriction, including   */
/*  without limitation the rights to use, copy, modify, merge, publish,  */
/*  distribute, sublicense, and/or sell copies of this work, and to      */
/*  permit persons to whom this work is furnished to do so, subject to   */
/*  the following conditions:                                            */
/*   1. The code must retain the above copyright notice, this list of    */
/*      conditions and the following disclaimer.                         */
/*   2. Any modifications must be clearly marked as such.                */
/*   3. Original authors' names are not deleted.                         */
/*   4. The authors' names are not used to endorse or promote products   */
/*      derived from this software without specific prior written        */
/*      permission.                                                      */
/*                                                                       */
/*  THE UNIVERSITY OF EDINBURGH AND THE CONTRIBUTORS TO THIS WORK        */
/*  DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING      */
/*  ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT   */
/*  SHALL THE UNIVERSITY OF EDINBURGH NOR THE CONTRIBUTORS BE LIABLE     */
/*  FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES    */
/*  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN   */
/*  AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,          */
/*  ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF       */
/*  THIS SOFTWARE.                                                       */
/*                                                                       */
/*************************************************************************/
/*                     Author :  Alan W Black                            */
/*                     Date   :  April 1996                              */
/*-----------------------------------------------------------------------*/
/*                    Token/Tokenizer class                              */
/*                                                                       */
/*=======================================================================*/

#ifndef __EST_TOKEN_H__
#define __EST_TOKEN_H__

#include <cstdio>

using namespace std;

#include "EST_String.h"
#include "EST_common.h"

// I can never really remember this so we'll define it here
/// The default whitespace characters
extern const EST_String EST_Token_Default_WhiteSpaceChars;
///
extern const EST_String EST_Token_Default_SingleCharSymbols;
///
extern const EST_String EST_Token_Default_PunctuationSymbols;
///
extern const EST_String EST_Token_Default_PrePunctuationSymbols;

/** This class is similar to \Ref{EST_String} but also maintains 
    the original punctuation and whitespace found around the 
    token.  

    \Ref{EST_Token}'s primary use is with \Ref{EST_TokenStream} class 
    which allows easy tokenizing of ascii files.  

    A token consists of four parts, any of which may be empty: a
    name, the actual token, preceding whitespace, preceding
    punctuation, the name and succeeding punctuation.  

    @author Alan W Black (awb@cstr.ed.ac.uk): April 1996
*/
class EST_Token {
  private:
    EST_String space;
    EST_String prepunc;
    EST_String pname;
    EST_String punc;
    int linenum;
    int linepos;
    int p_filepos;
    int p_quoted;

  public:
    ///
    EST_Token() {init();}
    ///
    EST_Token(const EST_String p) {init(); pname = p; }
    ///
    void init() {p_quoted=linenum=linepos=p_filepos=0;}
    
    /**@name Basic access to fields */
    //@{
    /// set token from a string
    void set_token(const EST_String &p) { pname = p; }
    ///
    void set_token(const char *p) { pname = p; }
    /// set whitespace of token.
    void set_whitespace(const EST_String &p) { space = p; }
    ///
    void set_whitespace(const char *p) { space = p; }
    /// set (post) punctuation of token.
    void set_punctuation(const EST_String &p) { punc = p; }
    /// 
    void set_punctuation(const char *p) { punc = p; }
    /// set prepunction
    void set_prepunctuation(const EST_String &p) { prepunc = p; }
    ///
    void set_prepunctuation(const char *p) { prepunc = p; }
    ///
    const EST_String &whitespace() { return space; }
    ///
    const EST_String &punctuation() { return punc; }
    ///
    const EST_String &prepunctuation() { return prepunc; }

    /**@name Access token as a string */
    //@{
    const EST_String &string() const { return String(); }
    /// Access token as a string
    const EST_String &S() const { return S(); }
    /// Access token as a string
    const EST_String &String() const { return pname; }
    /// For automatic coercion to \Ref{EST_String}
    operator EST_String() const { return String(); }
    //@}

    /**@name Access token as a int */
    //@{
    int Int(bool &valid) const { return String().Int(valid); }
    int Int() const { return String().Int(); }
    int I(bool &valid) const { return Int(valid); }
    int I() const { return Int(); }
    operator int() const { return Int(); }
    //@}

    /**@name Access token as a long */
    //@{
    long Long(bool &valid) const { return String().Long(valid); }
    long Long() const { return String().Long(); }
    long L(bool &valid) const { return Long(valid); }
    long L() const { return Long(); }
    operator long() const { return Long(); }
    //@}

    /**@name Access token as a float */
    //@{
    float Float(bool &valid) const { return String().Float(valid); }
    float Float() const { return String().Float(); }
    float F(bool &valid) const { return Float(valid); }
    float F() const { return Float(); }
    operator float() const { return Float(); }
    //@}

    /**@name Access token as a double */
    //@{
    double Double(bool &valid) const { return String().Double(valid); }
    double Double() const { return String().Double(); }
    double D(bool &valid) const { return Double(valid); }
    double D() const { return Double(); }
    operator double() const { return Double(); }
    //@}

    //@}
    //@{
    /// Note that this token was quoted (or not)
    void set_quoted(int q) { p_quoted = q; }
    /// TRUE is token was quoted
    int quoted() const { return p_quoted; }
    //@}
    ///
    void set_row(int r) { linenum = r; }
    ///
    void set_col(int c) { linepos = c; }
    /// Set file position in original \Ref{EST_TokenStream}
    void set_filepos(int c) { p_filepos = c; }
    /// Return lower case version of token name
    EST_String lstring() { return downcase(pname); }
    /// Return upper case version of token name
    EST_String ustring() { return upcase(pname); }
    /// Line number in original \Ref{EST_TokenStream}.
    int row(void) const { return linenum; }
    /// Line position in original \Ref{EST_TokenStream}.
    int col(void) const { return linepos; }
    /// file position in original \Ref{EST_TokenStream}.
    int filepos(void) const { return p_filepos; }

    /// A string describing current position, suitable for error messages
    const EST_String pos_description() const;

    ///
    friend ostream& operator << (ostream& s, const EST_Token &p);
    
    ///
    EST_Token & operator = (const EST_Token &a);
    ///
    EST_Token & operator = (const EST_String &a);
    ///
    int operator == (const EST_String &a) { return (pname == a); }
    ///
    int operator != (const EST_String &a) { return (pname != a); }
    ///
    int operator == (const char *a) { return (strcmp(pname,a)==0); }
    ///
    int operator != (const char *a) { return (strcmp(pname,a)!=0); }
};

enum EST_tokenstream_type {tst_none, tst_file, tst_pipe, tst_string, tst_istream}; 

/** A class that allows the reading of \Ref{EST_Token}s from a file
    stream, pipe or string.  It automatically tokenizes a file based on
    user definable whitespace and punctuation.

    The definitions of whitespace and punctuation are user definable.
    Also support for single character symbols is included.  Single
    character symbols {\em always} are treated as individual tokens
    irrespective of their white space context.  Also a quote
    mode can be used to read uqoted tokens.

    The setting of whitespace, pre and post punctuation, single character
    symbols and quote mode must be down (immediately) after opening
    the stream.

    There is no unget but peek provides look ahead of one token.
    
    Note there is an interesting issue about what to do about
    the last whitespace in the file.  Should it be ignored or should
    it be attached to a token with a name string of length zero.
    In unquoted mode the eof() will return TRUE if the next token name
    is empty (the mythical last token).  In quoted mode the last must
    be returned so eof will not be raised.

    @author Alan W Black (awb@cstr.ed.ac.uk): April 1996
*/
class EST_TokenStream{
 private:
    EST_tokenstream_type type;
    EST_String WhiteSpaceChars;
    EST_String SingleCharSymbols;
    EST_String PunctuationSymbols;
    EST_String PrePunctuationSymbols;
    EST_String Origin;
    FILE *fp;
    istream *is;
    int fd;
    char *buffer;
    int buffer_length;
    int pos;
    int linepos;
    int p_filepos;
    int getch(void);
    EST_TokenStream &getch(char &C);
    int peeked_charp;
    int peeked_char;       // ungot character 
    int peekch(void);
    int peeked_tokp;
    int eof_flag;
    int quotes;
    char quote;
    char escape;
    EST_Token current_tok;
    void default_values(void);
    /* local buffers to save reallocating */
    int tok_wspacelen;
    char *tok_wspace;
    int tok_stufflen;
    char *tok_stuff;
    int tok_prepuncslen;
    char *tok_prepuncs;
    int close_at_end;

    /* character class map */
    char p_table[256];
    bool p_table_wrong;

    /** This function is deliberately private so that you'll get a compilation
        error if you assign a token stream or pass it as an (non-reference)
        argument.  The problem with copying is that you need to copy the
        filedescriptiors too (which can't be done for pipes).  You probably
        don't really want a copy anyway and meant to pass it as a reference.
        If you really need this (some sort of clever look ahead) I am not
        sure what he consequences really are (or how portable they are).
        Pass the \Ref{EST_TokenStream} by reference instead.
    */
    EST_TokenStream(EST_TokenStream &s);

    void build_table();

    inline int getch_internal();
    inline int peekch_internal();
    inline int getpeeked_internal();
  public:
    ///
    EST_TokenStream();
    /// will close file if appropriate for type
    ~EST_TokenStream();
    //@{
    /// open a \Ref{EST_TokenStream} for a file.
    int open(const EST_String &filename);
    /// open a \Ref{EST_TokenStream} for an already opened file
    int open(FILE *ofp, int close_when_finished);
    /// open a \Ref{EST_TokenStream} for an already open istream
    int open(istream &newis);
    /// open a \Ref{EST_TokenStream} for string rather than a file
    int open_string(const EST_String &newbuffer);
    /// Close stream.
    void close(void);
    //@}
    /**@name stream access functions */
    //@{
    /// get next token in stream
    EST_TokenStream &get(EST_Token &t);
    /// get next token in stream
    EST_Token &get();
    /**@name  get the next token which must be the argument. */
    //@{
    EST_Token &must_get(EST_String expected, bool *ok);
    EST_Token &must_get(EST_String expected, bool &ok) 
	{ return must_get(expected, &ok); }
    EST_Token &must_get(EST_String expected) 
	{ return must_get(expected, (bool *)NULL); }
    //@}
    /// get up to {\tt s} in stream as a single token.
    EST_Token get_upto(const EST_String &s);
    /// get up to {\tt s} in end of line as a single token.
    EST_Token get_upto_eoln(void);
    /// peek at next token
    EST_Token &peek(void)
    {	if (!peeked_tokp) get();
	peeked_tokp = TRUE; return current_tok; }
    /// Reading binary data, (don't use peek() immediately beforehand)
    int fread(void *buff,int size,int nitems);
    //@}
    /**@name stream initialization functions */
    //@{
    /// set which characters are to be treated as whitespace
    void set_WhiteSpaceChars(const EST_String &ws) 
        { WhiteSpaceChars = ws; p_table_wrong=1;}
    /// set which characters are to be treated as single character symbols
    void set_SingleCharSymbols(const EST_String &sc) 
        { SingleCharSymbols = sc; p_table_wrong=1;}
    /// set which characters are to be treated as (post) punctuation
    void set_PunctuationSymbols(const EST_String &ps) 
        { PunctuationSymbols = ps; p_table_wrong=1;}
    /// set which characters are to be treated as (post) punctuation
    void set_PrePunctuationSymbols(const EST_String &ps) 
        { PrePunctuationSymbols = ps; p_table_wrong=1;}
    /// set characters to be used as quotes and escape, and set quote mode
    void set_quotes(char q, char e) { quotes = TRUE; quote = q; escape = e; p_table_wrong=1;}
    /// query quote mode
    int quoted_mode(void) { return quotes; }
    //@}
    /**@name miscellaneous */
    //@{
    /// returns line number of \Ref{EST_TokenStream}
    int linenum(void) const {return linepos;}
    /// end of file
    int eof()
       { return (eof_flag || ((!quotes) && (peek() == ""))); }
    /// end of line
    int eoln();
    /// current file position in \Ref{EST_TokenStream}
    int filepos(void) const { return (type == tst_string) ? pos : p_filepos; }
    /// tell, synonym for filepos
    int tell(void) const { return filepos(); }
    /// seek, reposition file pointer
    int seek(int position);
    int seek_end();
    /// Reset to start of file/string 
    int restart(void);
    /// A string describing current position, suitable for error messages
    const EST_String pos_description();
    /// The originating filename (if there is one)
    const EST_String filename() const { return Origin; }
    /// For the people who *need* the actual description (if possible)
    FILE *filedescriptor() { return (type == tst_file) ? fp : 0; }
    ///
    EST_TokenStream & operator >>(EST_Token &p);
    ///
    EST_TokenStream & operator >>(EST_String &p);
    ///
    friend ostream& operator <<(ostream& s, EST_TokenStream &p);
    //@}
};

/** Quote a string with given quotes and escape character
*/
EST_String quote_string(const EST_String &s,
			const EST_String &quote = "\"", 
			const EST_String &escape = "\\", 
			int force=0);

#endif // __EST_TOKEN_H__