language_models.py

   1 import string
   2 import norms
   3 import random
   4 import collections
   5 import unicodedata
   6 import itertools
   7 from math import log10
   8 import os
   9
  10
  11
  12 def datafile(name, sep='\t'):
  13     """Read key,value pairs from file.
  14     """
  15     with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), name), 'r') as f:
  16         for line in f:
  17             splits = line.split(sep)
  18             yield [splits[0], int(splits[1])]
  19
  20 english_counts = collections.Counter(dict(datafile('count_1l.txt')))
  21 normalised_english_counts = norms.normalise(english_counts)
  22
  23 english_bigram_counts = collections.Counter(dict(datafile('count_2l.txt')))
  24 normalised_english_bigram_counts = norms.normalise(english_bigram_counts)
  25
  26 english_trigram_counts = collections.Counter(dict(datafile('count_3l.txt')))
  27 normalised_english_trigram_counts = norms.normalise(english_trigram_counts)
  28
  29 with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'words.txt'), 'r') as f:
  30     keywords = [line.rstrip() for line in f]
  31
  32
  33 def weighted_choice(d):
  34     """Generate random item from a dictionary of item counts
  35     """
  36     target = random.uniform(0, sum(d.values()))
  37     cuml = 0.0
  38     for (l, p) in d.items():
  39         cuml += p
  40         if cuml > target:
  41             return l
  42     return None
  43
  44 def random_english_letter():
  45     """Generate a random letter based on English letter counts
  46     """
  47     return weighted_choice(normalised_english_counts)
  48
  49
  50 def ngrams(text, n):
  51     """Returns all n-grams of a text
  52
  53     >>> ngrams(sanitise('the quick brown fox'), 2) # doctest: +NORMALIZE_WHITESPACE
  54     ['th', 'he', 'eq', 'qu', 'ui', 'ic', 'ck', 'kb', 'br', 'ro', 'ow', 'wn',
  55      'nf', 'fo', 'ox']
  56     >>> ngrams(sanitise('the quick brown fox'), 4) # doctest: +NORMALIZE_WHITESPACE
  57     ['theq', 'hequ', 'equi', 'quic', 'uick', 'ickb', 'ckbr', 'kbro', 'brow',
  58      'rown', 'ownf', 'wnfo', 'nfox']
  59     """
  60     return [text[i:i+n] for i in range(len(text)-n+1)]
  61
  62
  63 class Pdist(dict):
  64     """A probability distribution estimated from counts in datafile.
  65     Values are stored and returned as log probabilities.
  66     """
  67     def __init__(self, data=[], estimate_of_missing=None):
  68         data1, data2 = itertools.tee(data)
  69         self.total = sum([d[1] for d in data1])
  70         for key, count in data2:
  71             self[key] = log10(count / self.total)
  72         self.estimate_of_missing = estimate_of_missing or (lambda k, N: 1./N)
  73     def __missing__(self, key):
  74         return self.estimate_of_missing(key, self.total)
  75
  76 def log_probability_of_unknown_word(key, N):
  77     """Estimate the probability of an unknown word.
  78     """
  79     return -log10(N * 10**((len(key) - 2) * 1.4))
  80
  81 Pw = Pdist(datafile('count_1w.txt'), log_probability_of_unknown_word)
  82 Pl = Pdist(datafile('count_1l.txt'), lambda _k, _N: 0)
  83 P2l = Pdist(datafile('count_2l.txt'), lambda _k, _N: 0)
  84 P3l = Pdist(datafile('count_3l.txt'), lambda _k, _N: 0)
  85
  86 def Pwords(words):
  87     """The Naive Bayes log probability of a sequence of words.
  88     """
  89     return sum(Pw[w.lower()] for w in words)
  90
  91 def Pletters(letters):
  92     """The Naive Bayes log probability of a sequence of letters.
  93     """
  94     return sum(Pl[l.lower()] for l in letters)
  95
  96 def Pbigrams(letters):
  97     """The Naive Bayes log probability of the bigrams formed from a sequence
  98     of letters.
  99     """
 100     return sum(P2l[p] for p in ngrams(letters, 2))
 101
 102 def Ptrigrams(letters):
 103     """The Naive Bayes log probability of the trigrams formed from a sequence
 104     of letters.
 105     """
 106     return sum(P3l[p] for p in ngrams(letters, 3))
 107
 108
 109 def cosine_distance_score(text):
 110     """Finds the dissimilarity of a text to English, using the cosine distance
 111     of the frequency distribution.
 112
 113     >>> cosine_distance_score('abcabc') # doctest: +ELLIPSIS
 114     0.73777...
 115     """
 116     # return norms.cosine_distance(english_counts,
 117     #     collections.Counter(sanitise(text)))
 118     return 1 - norms.cosine_similarity(english_counts,
 119         collections.Counter(sanitise(text)))
 120
 121
 122 if __name__ == "__main__":
 123     import doctest
 124     doctest.testmod()