7 #include <boost/tokenizer.hpp>
14 static const boost::char_separator<char>
AllSepatorList (
" .,;:|+-*/_=!@#$%`~^&(){}[]?'<>\"");
20 const boost::char_separator<char>& iSepatorList) {
25 typedef boost::tokenizer<boost::char_separator<char> > Tokeniser_T;
28 Tokeniser_T lTokens (iPhrase, iSepatorList);
29 for (Tokeniser_T::const_iterator tok_iter = lTokens.begin();
30 tok_iter != lTokens.end(); ++tok_iter) {
31 const std::string& lTerm = *tok_iter;
32 ioWordList.push_back (lTerm);
53 std::ostringstream oStr;
55 unsigned short idx = iWordList.size();
56 for (WordList_T::const_iterator itWord = iWordList.begin();
57 itWord != iWordList.end(); ++itWord, --idx) {
58 const std::string& lWord = *itWord;
void baseTokeniseStringIntoWordList(const std::string &iPhrase, WordList_T &ioWordList, const boost::char_separator< char > &iSepatorList)
static std::string createStringFromWordList(const WordList_T &iWordList)
std::list< Word_T > WordList_T
static void tokeniseStringIntoWordList(const TravelQuery_T &, WordList_T &)
static void tokeniseDocIntoWordList(const TravelQuery_T &, WordList_T &)
static const boost::char_separator< char > AllSepatorList(" .,;:|+-*/_=!@#$%`~^&(){}[]?'<>\"")
static const boost::char_separator< char > DocSepatorList(" ,-%")