21行Python代码实现拼写检查器

import re, collections

def words(text): return re.findall('[a-z]+', text.lower()) 

def train(features):
  model = collections.defaultdict(lambda: 1)
  for f in features:
    model[f] += 1
  return model

NWORDS = train(words(file('big.txt').read()))

alphabet = 'abcdefghijklmnopqrstuvwxyz'

def edits1(word):
  splits   = [(word[:i], word[i:]) for i in range(len(word) + 1)]
  deletes  = [a + b[1:] for a, b in splits if b]
  transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b)>1]
  replaces  = [a + c + b[1:] for a, b in splits for c in alphabet if b]
  inserts  = [a + c + b   for a, b in splits for c in alphabet]
  return set(deletes + transposes + replaces + inserts)

def known_edits2(word):
  return set(e2 for e1 in edits1(word) for e2 in edits1(e1) if e2 in NWORDS)

def known(words): return set(w for w in words if w in NWORDS)

def correct(word):
  candidates = known([word]) or known(edits1(word)) or known_edits2(word) or [word]
  return max(candidates, key=NWORDS.get)
correct函数是程序的入口，传进去错误拼写的单词会返回正确。如：

>>> correct("cpoy")
'copy'
>>> correct("engilsh")
'english'
>>> correct("sruprise")
'surprise'

除了这段代码外，作为机器学习的一部分，肯定还应该有大量的样本数据，准备了big.txt作为我们的样本数据。

背后原理

上面的代码是基于贝叶斯来实现的，事实上谷歌百度实现的拼写检查也是通过贝叶斯实现，不过肯定比这个复杂多了。
首先简单介绍一下背后的原理，如果读者之前了解过了，可以跳过这段。
给一个词，我们试图选取一个最可能的正确的的拼写建议（建议也可能就是输入的单词）。有时也不清楚（比如lates应该被更正为late或者latest？），我们用概率决定把哪一个作为建议。我们从跟原始词w相关的所有可能的正确拼写中找到可能性最大的那个拼写建议c：

argmaxc P(c|w)

argmaxc P(w|c) P(c) / P(w)

def words(text): return re.findall('[a-z]+', text.lower())

def train(features):
  model = collections.defaultdict(lambda: 1)
  for f in features:
    model[f] += 1
  return model
NWORDS = train(words(file('big.txt').read()))

def edits1(word):
  splits   = [(word[:i], word[i:]) for i in range(len(word) + 1)]
  deletes  = [a + b[1:] for a, b in splits if b]
  transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b)>1]
  replaces  = [a + c + b[1:] for a, b in splits for c in alphabet if b]
  inserts  = [a + c + b   for a, b in splits for c in alphabet]
  return set(deletes + transposes + replaces + inserts)

def known_edits2(word):
  return set(e2 for e1 in edits1(word) for e2 in edits1(e1) if e2 in NWORDS)

def known(words):
  return set(w for w in words if w in NWORDS)

def correct(word):
  candidates = known([word]) or known(edits1(word)) or known_edits2(word) or [word]
  return max(candidates, key=NWORDS.get)