/usr/include/mimetic/tokenizer.h is in libmimetic-dev 0.9.7-1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | /***************************************************************************
copyright : (C) 2002-2008 by Stefano Barbato
email : stefano@codesink.org
$Id: tokenizer.h,v 1.18 2008-10-07 11:44:38 tat Exp $
***************************************************************************/
#ifndef _MIMETIC_TOKENIZER_H_
#define _MIMETIC_TOKENIZER_H_
#include <iterator>
#include <algorithm>
#include <set>
#include <string>
#include <cstring>
namespace mimetic
{
template<typename value_type>
struct IsDelim: public std::unary_function<value_type,bool>
{
bool operator()(const value_type& val) const
{
return m_delims.count(val) != 0;
}
template<typename Container>
void setDelimList(const Container& cont)
{
typename Container::const_iterator bit, eit;
bit = cont.begin(), eit = cont.end();
for(; bit != eit; ++bit)
m_delims.insert(*bit);
}
template<typename Iterator>
void setDelimList(Iterator bit, Iterator eit)
{
for(; bit != eit; ++bit)
m_delims.insert(*bit);
}
void addDelim(const value_type& value)
{
m_delims.insert(value);
}
void removeDelim(const value_type& value)
{
m_delims.erase(value);
}
private:
std::set<value_type> m_delims;
};
template<>
struct IsDelim<char>: public std::unary_function<char, bool>
{
void setDelimList(const std::string& delims)
{
setDelimList(delims.begin(), delims.end());
}
template<typename Iterator>
void setDelimList(Iterator bit, Iterator eit)
{
memset(&m_lookup, 0, sizeof(m_lookup));
for(; bit != eit; ++bit)
m_lookup[(int)*bit] = 1;
}
bool operator()(unsigned char val) const
{
return m_lookup[val] != 0;
}
private:
char m_lookup[256];
};
/// Iterator tokenizer template class
template<class Iterator,typename value_type>
class ItTokenizer
{
public:
ItTokenizer(Iterator bit, Iterator eit)
: m_bit(bit), m_eit(eit), m_tok_eit(bit)
{
}
void setSource(Iterator bit, Iterator eit)
{
m_bit = bit;
m_eit = eit;
m_tok_eit = bit;
}
template<typename DelimCont>
void setDelimList(const DelimCont& cont)
{
m_delimPred.setDelimList(cont);
}
template<typename It>
void setDelimList(It bit, It eit)
{
m_delimPred.setDelimList(bit, eit);
}
template<typename DestCont>
bool next(DestCont& dst)
{
dst.erase(dst.begin(), dst.end());
if(m_tok_eit == m_eit)
return false;
m_tok_eit = std::find_if(m_bit, m_eit, m_delimPred);
m_matched = 0; // end of input
if(m_tok_eit != m_eit)
m_matched = *m_tok_eit; // matched delimiter
std::copy(m_bit, m_tok_eit, std::back_inserter<DestCont>(dst));
m_bit = (m_tok_eit != m_eit && ++m_tok_eit != m_eit ? m_tok_eit :m_eit);
return true;
}
const value_type& matched() const
{
return m_matched;
}
void addDelim(const value_type& value)
{
m_delimPred.addDelim(value);
}
void removeDelim(const value_type& value)
{
m_delimPred.removeDelim(value);
}
private:
Iterator m_bit, m_eit, m_tok_eit;
IsDelim<value_type> m_delimPred;
value_type m_matched;
};
/// char container tokenizer template class
template<typename Container>
struct ContTokenizer: public ItTokenizer<typename Container::const_iterator,typename Container::value_type>
{
typedef typename Container::value_type value_type;
typedef typename Container::iterator iterator;
typedef typename Container::const_iterator const_iterator;
// i want to be fast here so i don't want to copy "cont"
// so "cont" MUST be in scope for all following calls
// to next(...).
ContTokenizer(const Container* cont)
: ItTokenizer<const_iterator, value_type>(cont.begin(), cont.end())
{
}
template<typename DelimCont>
ContTokenizer(const Container* cont, const DelimCont& delims)
: ItTokenizer<const_iterator,value_type>(cont->begin(), cont->end())
{
setDelimList(delims);
}
void setSource(const Container* cont)
{
ItTokenizer<const_iterator,value_type>::setSource(cont->begin(), cont->end());
}
private:
ContTokenizer(const ContTokenizer&);
ContTokenizer& operator=(const ContTokenizer&);
};
/// std::string tokenizer
typedef ContTokenizer<std::string> StringTokenizer;
}
#endif
|