TextPreprocessor.cpp

//Removes all punctuation from text passed to it.
#include "TextPreprocessor.h"
#include <string>
#include <sstream>
#include <ctype.h>
#include <iostream>



using namespace std;

	
//loads the words from stream
vector<string> TextPreprocessor::getwords(string &line)
{
	string word;
	vector<string> words;
	
	istringstream iss(line, istringstream::in);
	while (iss >> word)
	{ 
    remove_punctuation(word);
		if (filterer.allowed(word))
		{
			//convert to required case and add it to the list
      process_case(word);
			if (word.size() > 0)
      {
        words.push_back(word);
      }
		}
		//else
		//{
		//	cout << "not allowing word: " << word << endl;
		//}

	}
	return words;
}

// removes punctuation from the start & end of the words
void TextPreprocessor::remove_punctuation(string &word)
{
	int i = 0;
	while (ispunct(word[i]) || iscntrl(word[i])) i++;
	int j = ((int)word.size() - 1);
	while (ispunct(word[j]) || iscntrl(word[i])) j--;
	
	if (i>0) 
	{ 
		word.erase(0,i);
		j-= i;
	}
	if ((j < ((word.size()-1))) && (j>0)) word.erase(j+1,word.size()-1);
}

// converts the case if necessary
void TextPreprocessor::process_case(string &word)
{
	if (settings.case_processing == cp_lower) 
	{
    std::transform(word.begin(), word.end(), word.begin(), 
                   (int(*)(int)) tolower);    
	}
	else if  (settings.case_processing == cp_upper)
	{
    std::transform(word.begin(), word.end(), word.begin(), 
                   (int(*)(int)) toupper);    
	} 
}