/usr/include/sunpinyin-2.0/slm/sim_slmbuilder.h is in libsunpinyin-dev 2.0.3-5ubuntu1.
This file is owned by root:root, with mode 0o644.
The actual contents of the file can be viewed below.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | /*
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
*
* Copyright (c) 2007 Sun Microsystems, Inc. All Rights Reserved.
*
* The contents of this file are subject to the terms of either the GNU Lesser
* General Public License Version 2.1 only ("LGPL") or the Common Development and
* Distribution License ("CDDL")(collectively, the "License"). You may not use this
* file except in compliance with the License. You can obtain a copy of the CDDL at
* http://www.opensource.org/licenses/cddl1.php and a copy of the LGPLv2.1 at
* http://www.opensource.org/licenses/lgpl-license.php. See the License for the
* specific language governing permissions and limitations under the License. When
* distributing the software, include this License Header Notice in each file and
* include the full text of the License in the License file as well as the
* following notice:
*
* NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE
* (CDDL)
* For Covered Software in this distribution, this License shall be governed by the
* laws of the State of California (excluding conflict-of-law provisions).
* Any litigation relating to this License shall be subject to the jurisdiction of
* the Federal Courts of the Northern District of California and the state courts
* of the State of California, with venue lying in Santa Clara County, California.
*
* Contributor(s):
*
* If you wish your version of this file to be governed by only the CDDL or only
* the LGPL Version 2.1, indicate your decision by adding "[Contributor]" elects to
* include this software in this distribution under the [CDDL or LGPL Version 2.1]
* license." If you don't indicate a single choice of license, a recipient has the
* option to distribute your version of this file under either the CDDL or the LGPL
* Version 2.1, or to extend the choice of license to its licensees as provided
* above. However, if you add LGPL Version 2.1 code and therefore, elected the LGPL
* Version 2 license, then the option applies only if the new code is made subject
* to such option by the copyright holder.
*/
#ifndef _SIM_SLM_BUILDER_H
#define _SIM_SLM_BUILDER_H
#include "../portability.h"
#include "sim_slm.h"
class CSlmDiscounter;
class CSlmBuilder {
public:
static const int SLM_MAX_R=16;
typedef CSIMSlm::FREQ_TYPE FREQ_TYPE;
typedef CSIMSlm::PR_TYPE PR_TYPE;
typedef CSIMSlm::TNode TNode;
typedef CSIMSlm::TLeaf TLeaf;
public:
CSlmBuilder()
: m_nWord(0), nlevel(0), level(NULL), cut(NULL), discounter(NULL),
nr(NULL), breaker(), m_excludes(), bUseLogPr(0) { }
~CSlmBuilder()
{ Close(); }
void Create(int n);
void SetNumberOfWord(int nWord) { this->m_nWord = nWord; }
void SetCut(FREQ_TYPE threshold[]);
void SetDiscounter(CSlmDiscounter* dis[]);
void SetBreakerIds(int nId, TSIMWordId brks[]);
void SetExcludeIds(int nId, TSIMWordId excludes[]);
void SetUseLogPr(int bUse)
{ bUseLogPr = bUse; }
void AddNGram(TSIMWordId* ngram, FREQ_TYPE fr);
void Build();
void Write(FILE* out);
void Close();
//get pr(w[n-1] | w[0]...w[n-2]) on constructed partial model (low levels)
double getPr(int n, TSIMWordId* w);
public:
typedef std::vector<TNode> TNodeLevel;
typedef std::vector<TLeaf> TLeafLevel;
typedef TNodeLevel::iterator TNodeIterator;
typedef TLeafLevel::iterator TLeafIterator;
protected:
bool isBreakId(TSIMWordId id);
bool isExcludeId(TSIMWordId id);
void CountNr();
void AppendTails();
void Cut();
void Discount();
void CalcBOW();
void*FindChild(int lvl, TNode* root, TSIMWordId id);
int CutNodeLevel(TNodeIterator pfirst, TNodeIterator plast,
TNodeIterator chfirst, TNodeIterator chlast, int thred);
int CutLeafLevel(TNodeIterator pfirst, TNodeIterator plast,
TLeafIterator chfirst, TLeafIterator chlast, int thred);
private:
int nlevel, bUseLogPr;
void** level;
//level[0] is psudeo root level, level[1] is unigram level, ..., all are vector type
int m_nWord;
FREQ_TYPE* cut; // cut[1] is not cut threshold for 1-gram, ...
CSlmDiscounter** discounter; // discounter[1] is for 1-gram...
FREQ_TYPE (*nr)[SLM_MAX_R];//nr[1][SLM_MAX_R] is for 1-gram...
std::vector<TSIMWordId> breaker;
std::vector<TSIMWordId> m_excludes;
};
class CSlmDiscounter {
public:
// n is array size, nr is FREQ_TYPE[n], nr[0] is corpuse size,or sigma r*nr;
// nr[1] is number of ngram items with freq 1, ...
virtual void init(int n, CSlmBuilder::FREQ_TYPE *nr) = 0;
// freq is the ngram frequence, not the conditional pr
virtual double discount(int freq) = 0;
virtual const char* getName() = 0;
};
//Good-Turing discount
class CSlmGTDiscounter : public CSlmDiscounter {
public:
CSlmGTDiscounter(int threshold=10, double highfreq_discount=0.95) : thres(threshold), hd(highfreq_discount), dis(NULL) {}
virtual void init(int n, CSlmBuilder::FREQ_TYPE *nr);
virtual double discount(int freq);
virtual const char* getName()
{ return "Good-Turing"; }
protected:
int thres;
double hd;
double *dis;
};
class CSlmAbsoluteDiscounter : public CSlmDiscounter {
public:
CSlmAbsoluteDiscounter(double substract = 0.0) : c(substract) {}
//c == 0 mean this value should be count according to r[]
virtual void init(int n, CSlmBuilder::FREQ_TYPE *nr);
virtual double discount(int freq); // return freq - c
virtual const char* getName()
{ return "Absolution"; }
protected:
double c;
};
class CSlmLinearDiscounter : public CSlmDiscounter {
public:
CSlmLinearDiscounter(double shrink = 0.0) : dis(shrink) {}
//dis == 0 mean this value should be count according to r[]
virtual void init(int n, CSlmBuilder::FREQ_TYPE *nr);
virtual double discount(int freq); // return freq * dis
virtual const char* getName()
{ return "Linear"; }
protected:
double dis;
};
#endif
|