X-Git-Url: https://git.njae.me.uk/?a=blobdiff_plain;f=language_models.py;fp=language_models.py;h=d45738657a356a38c5139838da665044fe9257e6;hb=49dc272d2fc91e7340e56e9e7b96da6ab63514bb;hp=8c98a2e27906ed959b5820f2ef58f38f5dbd157a;hpb=7bfededb9542780a13f38527c8ff21e89d6c08af;p=cipher-tools.git diff --git a/language_models.py b/language_models.py index 8c98a2e..d457386 100644 --- a/language_models.py +++ b/language_models.py @@ -3,33 +3,76 @@ import norms import random import collections import unicodedata +import itertools +from math import log10 -english_counts = collections.defaultdict(int) -with open('count_1l.txt', 'r') as f: - for line in f: - (letter, count) = line.split("\t") - english_counts[letter] = int(count) +def letters(text): + """Remove all non-alphabetic characters from a text + >>> letters('The Quick') + 'TheQuick' + >>> letters('The Quick BROWN fox jumped! over... the (9lazy) DOG') + 'TheQuickBROWNfoxjumpedoverthelazyDOG' + """ + return ''.join([c for c in text if c in string.ascii_letters]) + +def unaccent(text): + """Remove all accents from letters. + It does this by converting the unicode string to decomposed compatability + form, dropping all the combining accents, then re-encoding the bytes. + + >>> unaccent('hello') + 'hello' + >>> unaccent('HELLO') + 'HELLO' + >>> unaccent('héllo') + 'hello' + >>> unaccent('héllö') + 'hello' + >>> unaccent('HÉLLÖ') + 'HELLO' + """ + return unicodedata.normalize('NFKD', text).\ + encode('ascii', 'ignore').\ + decode('utf-8') + +def sanitise(text): + """Remove all non-alphabetic characters and convert the text to lowercase + + >>> sanitise('The Quick') + 'thequick' + >>> sanitise('The Quick BROWN fox jumped! over... the (9lazy) DOG') + 'thequickbrownfoxjumpedoverthelazydog' + >>> sanitise('HÉLLÖ') + 'hello' + """ + # sanitised = [c.lower() for c in text if c in string.ascii_letters] + # return ''.join(sanitised) + return letters(unaccent(text)).lower() + + +def datafile(name, sep='\t'): + """Read key,value pairs from file. + """ + with open(name, 'r') as f: + for line in f: + splits = line.split(sep) + yield [splits[0], int(splits[1])] + +english_counts = collections.Counter(dict(datafile('count_1l.txt'))) normalised_english_counts = norms.normalise(english_counts) -english_bigram_counts = collections.defaultdict(int) -with open('count_2l.txt', 'r') as f: - for line in f: - (bigram, count) = line.split("\t") - english_bigram_counts[bigram] = int(count) +english_bigram_counts = collections.Counter(dict(datafile('count_2l.txt'))) normalised_english_bigram_counts = norms.normalise(english_bigram_counts) -english_trigram_counts = collections.defaultdict(int) -with open('count_3l.txt', 'r') as f: - for line in f: - (trigram, count) = line.split("\t") - english_trigram_counts[trigram] = int(count) +english_trigram_counts = collections.Counter(dict(datafile('count_3l.txt'))) normalised_english_trigram_counts = norms.normalise(english_trigram_counts) with open('words.txt', 'r') as f: keywords = [line.rstrip() for line in f] + def weighted_choice(d): - """Generate a set of random items from a dictionary of item counts + """Generate random item from a dictionary of item counts """ target = random.uniform(0, sum(d.values())) cuml = 0.0 @@ -45,48 +88,48 @@ def random_english_letter(): return weighted_choice(normalised_english_counts) -def letters(text): - """Remove all non-alphabetic characters from a text - >>> letters('The Quick') - 'TheQuick' - >>> letters('The Quick BROWN fox jumped! over... the (9lazy) DOG') - 'TheQuickBROWNfoxjumpedoverthelazyDOG' +class Pdist(dict): + """A probability distribution estimated from counts in datafile. + Values are stored and returned as log probabilities. """ - return ''.join([c for c in text if c in string.ascii_letters]) + def __init__(self, data=[], estimate_of_missing=None): + data1, data2 = itertools.tee(data) + self.total = sum([d[1] for d in data1]) + for key, count in data2: + self[key] = log10(count / self.total) + self.estimate_of_missing = estimate_of_missing or (lambda k, N: 1./N) + def __missing__(self, key): + return self.estimate_of_missing(key, self.total) -def unaccent(text): - """Remove all accents from letters. - It does this by converting the unicode string to decomposed compatability - form, dropping all the combining accents, then re-encoding the bytes. - - >>> unaccent('hello') - 'hello' - >>> unaccent('HELLO') - 'HELLO' - >>> unaccent('héllo') - 'hello' - >>> unaccent('héllö') - 'hello' - >>> unaccent('HÉLLÖ') - 'HELLO' - """ - return unicodedata.normalize('NFKD', text).\ - encode('ascii', 'ignore').\ - decode('utf-8') +def log_probability_of_unknown_word(key, N): + """Estimate the probability of an unknown word. + """ + return -log10(N * 10**((len(key) - 2) * 1.4)) -def sanitise(text): - """Remove all non-alphabetic characters and convert the text to lowercase - - >>> sanitise('The Quick') - 'thequick' - >>> sanitise('The Quick BROWN fox jumped! over... the (9lazy) DOG') - 'thequickbrownfoxjumpedoverthelazydog' - >>> sanitise('HÉLLÖ') - 'hello' +Pw = Pdist(datafile('count_1w.txt'), log_probability_of_unknown_word) +Pl = Pdist(datafile('count_1l.txt'), lambda _k, _N: 0) + +def Pwords(words): + """The Naive Bayes log probability of a sequence of words. """ - # sanitised = [c.lower() for c in text if c in string.ascii_letters] - # return ''.join(sanitised) - return letters(unaccent(text)).lower() + return sum(Pw[w.lower()] for w in words) + +def Pletters(letters): + """The Naive Bayes log probability of a sequence of letters. + """ + return sum(Pl[l.lower()] for l in letters) + + + +def cosine_distance_score(text): + """Finds the dissimilarity of a text to English, using the cosine distance + of the frequency distribution. + + >>> cosine_distance_score('abcabc') # doctest: +ELLIPSIS + 0.370847405... + """ + return norms.cosine_distance(english_counts, + collections.Counter(sanitise(text))) if __name__ == "__main__":