我有两个文本文件（1）坏词样本和（2）好词汇样本。现在我已执行最近邻居分类，其中找到了新单词将被分类为好或坏。我想要了解如何使用我现有的代码来解决这个问题。感谢
class Words_Works():
def __init__(self):
self.all_texts = {}
self.categories = {}
self.knn_results = {}
self.stop_words = ['and','the','i','am','he','she','his',
'me','my','a','at','in','you','your',
'of','to','this','that','him','her',
'they','is','it','can','for','into',
'as','with','we','us','them','a',
'it', 'on', 'so', 'too','k','the',
'but', 'are','though'
'very', 'here', 'even', 'from',
'then', 'than']
self.leaf_words = ['s', 'es', 'ed', 'er', 'ly', 'ing']
def add_category(self,f,cat_name):
f_in = open(f)
self.text = f_in.read().lower()
f_in.close()
self.wordify()
self.unstopify()
self.unleafify()
self.categories[cat_name] = {}
for item in self.unleaf:
if self.categories[cat_name].has_key(item):
self.categories[cat_name][item] += 1
else:
self.categories[cat_name][item] = 1
def load_categories(self):
try:
cat_db = open('tweetCategory.txt','rb')
self.categories = cPickle.load(cat_db)
cat_db.close()
print 'File successfully loaded from categories db'
except:
print 'File not loaded from categories_db'
# Finds the levenshtein's distance
def levenshtein_distance(first, second):
"""Find the Levenshtein distance between two strings."""
if len(first) > len(second):
first, second = second, first
if len(second) == 0:
return len(first)
first_length = len(first) + 1
second_length = len(second) + 1
distance_matrix = [[0] * second_length for x in range(first_length)]
for i in range(first_length):
distance_matrix[i][0] = i
for j in range(second_length):
distance_matrix[0][j]=j
for i in xrange(1, first_length):
for j in range(1, second_length):
deletion = distance_matrix[i-1][j] + 1
insertion = distance_matrix[i][j-1] + 1
substitution = distance_matrix[i-1][j-1]
if first[i-1] != second[j-1]:
substitution += 1
distance_matrix[i][j] = min(insertion, deletion, substitution)
return distance_matrix[first_length-1][second_length-1]
def add_text(self,f):
f_in = open(f)
self.text = f_in.read().lower()
f_in.close()
self.wordify()
self.unstopify()
self.unleafify()
self.all_texts[f] = {}
for item in self.unleaf:
if self.all_texts[f].has_key(item):
self.all_texts[f][item] += 1
else:
self.all_texts[f][item] = 1
def save_categories(self):
cat_db = open('tweetCategory.txt','wb')
cPickle.dump(cat_db,self.categories,-1)
cat_db.close()
def unstopify(self):
self.unstop = [item for item in self.words if item not in self.stop_words]
def unleafify(self):
self.unleaf = self.unstop[:]
for leaf in self.leaf_words:
leaf_len = len(leaf)
leaf_pattern = re.compile('%s$' % leaf)
for i in range(len(self.unleaf)):
if leaf_pattern.findall(self.unleaf[i]):
self.unleaf[i] = self.unleaf[i][:-leaf_len]
def wordify(self):
words_pattern = re.compile('//w+')
self.words = words_pattern.findall(self.text)
def knn_calc(self):
for text in self.all_texts.keys():
self.knn_results[text] = {}
for category in self.categories.keys():
self.knn_results[text][category] = {}
iterations = 0
distance = 0
for word in self.all_texts[text].keys():
if word in self.categories[text].keys():
distance = levenshtein_distance(text,category)
self.knn_results[text][category]['Knn Distance'] = distance
self.knn_results [text][category]['Knn Iterations'] = iterations
def knn(self):
for text in self.all_texts.keys():
Result = None
for category in self.categories.keys():
if not result or self.knn_results[text][category]['Knn Distance'] < result:
knn = category
distance = self.knn_results[text][category]['Knn Distance']
iterations = self.knn_results[text][category]['Knn Iterations']
print 'File:',text
print 'Knn:',category
print 'Distance :', distance
print 'Iterations :', iterations
print 'End of nearest neighbour search'
测试案例试用：
mywork = Words_Works()
positive = 'positive.txt'
mywork.add_category(positive, 'Positive Tweets') # Adding as category
negative = 'negative.txt'
mywork.add_category(negative, 'Negative Tweets')
neutral = 'neutral.txt'
mywork.add_category(neutral, 'Neutral Tweets')
for category in mywork.categories.keys(): # Print categories
print category
print mywork.categories[category]
print
print
txts = ('samplegood.txt', 'samplebad.txt') # Creating list of files to
for text in txts: # Adding them
mywork.add_text(text)
for text in mywork.all_texts.keys(): # Print texts
print text
print mywork.all_texts[text]
print
print
mywork.knn_calc() # calculate knn
for files in mywork.knn_results.keys(): # print detailed results
print files
for category in mywork.knn_results[files].keys():
print category
print mywork.knn_results[files][category]
print
print
mywork.knn() # Display results
答案 0 :(得分：0)
使用编辑距离，您不在欧几里德空间。 http://en.wikipedia.org/wiki/Edit_distance
答案 1 :(得分：0)
两条建议：首先，如@YvesDaoust所述，您应该使用编辑距离，也称为Levenshtein距离。您可以在python-Levenshtein package。
中找到它其次，使用标准库中的unittest或doctest库来测试代码。使用保存在外部文件中的示例来测试代码是一个坏主意，因为没有访问这些文件的第三个人（例如我们）无法知道输入是什么;避免打印输出并手动检查它，因为这很慢，容易出错，而且其他人也无法审查。