8 #include <boost/tokenizer.hpp>
22 typedef boost::tokenizer<boost::char_separator<char> > Tokeniser_T;
25 const boost::char_separator<char>
26 lSepatorList(
" .,;:|+-*/_=!@#$%`~^&(){}[]?'<>\"");
29 Tokeniser_T lTokens (iPhrase, lSepatorList);
30 for (Tokeniser_T::const_iterator tok_iter = lTokens.begin();
31 tok_iter != lTokens.end(); ++tok_iter) {
32 const std::string& lTerm = *tok_iter;
33 ioWordList.push_back (lTerm);
39 const unsigned short iSplitIdx,
40 const bool iFromBeginningFlag) {
41 std::ostringstream oStr;
44 unsigned short idx = 0;
45 WordList_T::const_iterator itWord = iWordList.begin();
46 for ( ; itWord != iWordList.end(); ++itWord, ++idx) {
48 if (iFromBeginningFlag ==
true) {
53 if (iSplitIdx != 0 && idx >= iSplitIdx) {
62 const std::string& lWord = *itWord;
70 if (iSplitIdx == 0 || idx >= iSplitIdx) {
77 if (iFromBeginningFlag ==
true) {
82 assert (iFromBeginningFlag ==
false);
85 for ( ; itWord != iWordList.end(); ++itWord, ++idx) {
89 if (idx > iSplitIdx) {
93 const std::string& lWord = *itWord;
std::string createStringFromWordList(const WordList_T &iWordList, const unsigned short iSplitIdx, const bool iFromBeginningFlag)
std::list< Word_T > WordList_T
void tokeniseStringIntoWordList(const std::string &iPhrase, WordList_T &ioWordList)