cipherbreak.py

   1 import string
   2 import collections
   3 import norms
   4 import logging
   5 from itertools import zip_longest, cycle, permutations
   6 from segment import segment
   7 from multiprocessing import Pool
   8 from math import log10
   9
  10 import matplotlib.pyplot as plt
  11
  12 from cipher import *
  13 from language_models import *
  14
  15 # To time a run:
  16 #
  17 # import timeit
  18 # c5a = open('2012/5a.ciphertext', 'r').read()
  19 # timeit.timeit('keyword_break(c5a)', setup='gc.enable() ; from __main__ import c5a ; from cipher import keyword_break', number=1)
  20 # timeit.repeat('keyword_break_mp(c5a, chunksize=500)', setup='gc.enable() ; from __main__ import c5a ; from cipher import keyword_break_mp', repeat=5, number=1)
  21
  22 transpositions = collections.defaultdict(list)
  23 for word in keywords:
  24     transpositions[transpositions_of(word)] += [word]
  25
  26 def frequencies(text):
  27     """Count the number of occurrences of each character in text
  28
  29     >>> sorted(frequencies('abcdefabc').items())
  30     [('a', 2), ('b', 2), ('c', 2), ('d', 1), ('e', 1), ('f', 1)]
  31     >>> sorted(frequencies('the quick brown fox jumped over the lazy ' \
  32          'dog').items()) # doctest: +NORMALIZE_WHITESPACE
  33     [(' ', 8), ('a', 1), ('b', 1), ('c', 1), ('d', 2), ('e', 4), ('f', 1),
  34      ('g', 1), ('h', 2), ('i', 1), ('j', 1), ('k', 1), ('l', 1), ('m', 1),
  35      ('n', 1), ('o', 4), ('p', 1), ('q', 1), ('r', 2), ('t', 2), ('u', 2),
  36      ('v', 1), ('w', 1), ('x', 1), ('y', 1), ('z', 1)]
  37     >>> sorted(frequencies('The Quick BROWN fox jumped! over... the ' \
  38          '(9lazy) DOG').items()) # doctest: +NORMALIZE_WHITESPACE
  39     [(' ', 8), ('!', 1), ('(', 1), (')', 1), ('.', 3), ('9', 1), ('B', 1),
  40      ('D', 1), ('G', 1), ('N', 1), ('O', 2), ('Q', 1), ('R', 1), ('T', 1),
  41      ('W', 1), ('a', 1), ('c', 1), ('d', 1), ('e', 4), ('f', 1), ('h', 2),
  42      ('i', 1), ('j', 1), ('k', 1), ('l', 1), ('m', 1), ('o', 2), ('p', 1),
  43      ('r', 1), ('t', 1), ('u', 2), ('v', 1), ('x', 1), ('y', 1), ('z', 1)]
  44     >>> sorted(frequencies(sanitise('The Quick BROWN fox jumped! over... ' \
  45          'the (9lazy) DOG')).items()) # doctest: +NORMALIZE_WHITESPACE
  46     [('a', 1), ('b', 1), ('c', 1), ('d', 2), ('e', 4), ('f', 1), ('g', 1),
  47      ('h', 2), ('i', 1), ('j', 1), ('k', 1), ('l', 1), ('m', 1), ('n', 1),
  48      ('o', 4), ('p', 1), ('q', 1), ('r', 2), ('t', 2), ('u', 2), ('v', 1),
  49      ('w', 1), ('x', 1), ('y', 1), ('z', 1)]
  50     >>> frequencies('abcdefabcdef')['x']
  51     0
  52     """
  53     #counts = collections.defaultdict(int)
  54     #for c in text:
  55     #    counts[c] += 1
  56     #return counts
  57     return collections.Counter(c for c in text)
  58
  59
  60 def caesar_break(message, fitness=Pletters):
  61     """Breaks a Caesar cipher using frequency analysis
  62
  63     >>> caesar_break('ibxcsyorsaqcheyklxivoexlevmrimwxsfiqevvmihrsasrxliwyrh' \
  64           'ecjsppsamrkwleppfmergefifvmhixscsymjcsyqeoixlm') # doctest: +ELLIPSIS
  65     (4, -130.849890899...)
  66     >>> caesar_break('wxwmaxdgheetgwuxztgptedbgznitgwwhpguxyhkxbmhvvtlbhgtee' \
  67           'raxlmhiixweblmxgxwmhmaxybkbgztgwztsxwbgmxgmert') # doctest: +ELLIPSIS
  68     (19, -128.82516920...)
  69     >>> caesar_break('yltbbqnqnzvguvaxurorgenafsbezqvagbnornfgsbevpnaabjurer' \
  70           'svaquvzyvxrnznazlybequrvfohgriraabjtbaruraprur') # doctest: +ELLIPSIS
  71     (13, -126.25233502...)
  72     """
  73     sanitised_message = sanitise(message)
  74     best_shift = 0
  75     best_fit = float('-inf')
  76     for shift in range(26):
  77         plaintext = caesar_decipher(sanitised_message, shift)
  78         fit = fitness(plaintext)
  79         logger.debug('Caesar break attempt using key {0} gives fit of {1} '
  80                       'and decrypt starting: {2}'.format(shift, fit, plaintext[:50]))
  81         if fit > best_fit:
  82             best_fit = fit
  83             best_shift = shift
  84     logger.info('Caesar break best fit: key {0} gives fit of {1} and '
  85                 'decrypt starting: {2}'.format(best_shift, best_fit,
  86                     caesar_decipher(sanitised_message, best_shift)[:50]))
  87     return best_shift, best_fit
  88
  89 def affine_break(message,
  90                  metric=norms.euclidean_distance,
  91                  target_counts=normalised_english_counts,
  92                  message_frequency_scaling=norms.normalise):
  93     """Breaks an affine cipher using frequency analysis
  94
  95     >>> affine_break('lmyfu bkuusd dyfaxw claol psfaom jfasd snsfg jfaoe ls ' \
  96           'omytd jlaxe mh jm bfmibj umis hfsul axubafkjamx. ls kffkxwsd jls ' \
  97           'ofgbjmwfkiu olfmxmtmwaokttg jlsx ls kffkxwsd jlsi zg tsxwjl. jlsx ' \
  98           'ls umfjsd jlsi zg hfsqysxog. ls dmmdtsd mx jls bats mh bkbsf. ls ' \
  99           'bfmctsd kfmyxd jls lyj, mztanamyu xmc jm clm cku tmmeaxw kj lai ' \
 100           'kxd clm ckuxj.') # doctest: +ELLIPSIS
 101     ((15, 22, True), 0.0598745365924...)
 102     """
 103     sanitised_message = sanitise(message)
 104     best_multiplier = 0
 105     best_adder = 0
 106     best_one_based = True
 107     best_fit = float("inf")
 108     for one_based in [True, False]:
 109         for multiplier in range(1, 26, 2):
 110             for adder in range(26):
 111                 plaintext = affine_decipher(sanitised_message,
 112                                             multiplier, adder, one_based)
 113                 counts = message_frequency_scaling(frequencies(plaintext))
 114                 fit = metric(target_counts, counts)
 115                 logger.debug('Affine break attempt using key {0}x+{1} ({2}) '
 116                              'gives fit of {3} and decrypt starting: {4}'.
 117                              format(multiplier, adder, one_based, fit,
 118                                     plaintext[:50]))
 119                 if fit < best_fit:
 120                     best_fit = fit
 121                     best_multiplier = multiplier
 122                     best_adder = adder
 123                     best_one_based = one_based
 124     logger.info('Affine break best fit with key {0}x+{1} ({2}) gives fit of {3} '
 125                 'and decrypt starting: {4}'.format(
 126                     best_multiplier, best_adder, best_one_based, best_fit,
 127                     affine_decipher(sanitised_message, best_multiplier,
 128                         best_adder, best_one_based)[:50]))
 129     return (best_multiplier, best_adder, best_one_based), best_fit
 130
 131 def keyword_break(message,
 132                   wordlist=keywords,
 133                   metric=norms.euclidean_distance,
 134                   target_counts=normalised_english_counts,
 135                   message_frequency_scaling=norms.normalise):
 136     """Breaks a keyword substitution cipher using a dictionary and
 137     frequency analysis
 138
 139     >>> keyword_break(keyword_encipher('this is a test message for the ' \
 140           'keyword decipherment', 'elephant', 1), \
 141           wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS
 142     (('elephant', 1), 0.1066453448861...)
 143     """
 144     best_keyword = ''
 145     best_wrap_alphabet = True
 146     best_fit = float("inf")
 147     for wrap_alphabet in range(3):
 148         for keyword in wordlist:
 149             plaintext = keyword_decipher(message, keyword, wrap_alphabet)
 150             counts = message_frequency_scaling(frequencies(plaintext))
 151             fit = metric(target_counts, counts)
 152             logger.debug('Keyword break attempt using key {0} (wrap={1}) '
 153                          'gives fit of {2} and decrypt starting: {3}'.format(
 154                              keyword, wrap_alphabet, fit,
 155                              sanitise(plaintext)[:50]))
 156             if fit < best_fit:
 157                 best_fit = fit
 158                 best_keyword = keyword
 159                 best_wrap_alphabet = wrap_alphabet
 160     logger.info('Keyword break best fit with key {0} (wrap={1}) gives fit of '
 161                 '{2} and decrypt starting: {3}'.format(best_keyword,
 162                     best_wrap_alphabet, best_fit, sanitise(
 163                         keyword_decipher(message, best_keyword,
 164                                          best_wrap_alphabet))[:50]))
 165     return (best_keyword, best_wrap_alphabet), best_fit
 166
 167 def keyword_break_mp(message,
 168                      wordlist=keywords,
 169                      metric=norms.euclidean_distance,
 170                      target_counts=normalised_english_counts,
 171                      message_frequency_scaling=norms.normalise,
 172                      chunksize=500):
 173     """Breaks a keyword substitution cipher using a dictionary and
 174     frequency analysis
 175
 176     >>> keyword_break_mp(keyword_encipher('this is a test message for the ' \
 177           'keyword decipherment', 'elephant', 1), \
 178           wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS
 179     (('elephant', 1), 0.106645344886...)
 180     """
 181     with Pool() as pool:
 182         helper_args = [(message, word, wrap, metric, target_counts,
 183                         message_frequency_scaling)
 184                        for word in wordlist for wrap in range(3)]
 185         # Gotcha: the helper function here needs to be defined at the top level
 186         #   (limitation of Pool.starmap)
 187         breaks = pool.starmap(keyword_break_worker, helper_args, chunksize)
 188         return min(breaks, key=lambda k: k[1])
 189
 190 def keyword_break_worker(message, keyword, wrap_alphabet, metric, target_counts,
 191                       message_frequency_scaling):
 192     plaintext = keyword_decipher(message, keyword, wrap_alphabet)
 193     counts = message_frequency_scaling(frequencies(plaintext))
 194     fit = metric(target_counts, counts)
 195     logger.debug('Keyword break attempt using key {0} (wrap={1}) gives fit of '
 196                  '{2} and decrypt starting: {3}'.format(keyword,
 197                      wrap_alphabet, fit, sanitise(plaintext)[:50]))
 198     return (keyword, wrap_alphabet), fit
 199
 200 def scytale_break(message,
 201                   metric=norms.euclidean_distance,
 202                   target_counts=normalised_english_bigram_counts,
 203                   message_frequency_scaling=norms.normalise):
 204     """Breaks a Scytale cipher
 205
 206     >>> scytale_break('tfeulchtrtteehwahsdehneoifeayfsondmwpltmaoalhikotoere' \
 207            'dcweatehiplwxsnhooacgorrcrcraotohsgullasenylrendaianeplscdriioto' \
 208            'aek') # doctest: +ELLIPSIS
 209     (6, 0.092599933059...)
 210     """
 211     best_key = 0
 212     best_fit = float("inf")
 213     ngram_length = len(next(iter(target_counts.keys())))
 214     for key in range(1, 20):
 215         if len(message) % key == 0:
 216             plaintext = scytale_decipher(message, key)
 217             counts = message_frequency_scaling(frequencies(
 218                          ngrams(sanitise(plaintext), ngram_length)))
 219             fit = metric(target_counts, counts)
 220             logger.debug('Scytale break attempt using key {0} gives fit of '
 221                          '{1} and decrypt starting: {2}'.format(key,
 222                              fit, sanitise(plaintext)[:50]))
 223             if fit < best_fit:
 224                 best_fit = fit
 225                 best_key = key
 226     logger.info('Scytale break best fit with key {0} gives fit of {1} and '
 227                 'decrypt starting: {2}'.format(best_key, best_fit,
 228                     sanitise(scytale_decipher(message, best_key))[:50]))
 229     return best_key, best_fit
 230
 231
 232 def column_transposition_break_mp(message,
 233                      translist=transpositions,
 234                      metric=norms.euclidean_distance,
 235                      target_counts=normalised_english_bigram_counts,
 236                      message_frequency_scaling=norms.normalise,
 237                      chunksize=500):
 238     """Breaks a column transposition cipher using a dictionary and
 239     n-gram frequency analysis
 240     """
 241     # >>> column_transposition_break_mp(column_transposition_encipher(sanitise( \
 242     #         "It is a truth universally acknowledged, that a single man in \
 243     #          possession of a good fortune, must be in want of a wife. However \
 244     #          little known the feelings or views of such a man may be on his \
 245     #          first entering a neighbourhood, this truth is so well fixed in the \
 246     #          minds of the surrounding families, that he is considered the \
 247     #          rightful property of some one or other of their daughters."), \
 248     #     'encipher'), \
 249     #     translist={(2, 0, 5, 3, 1, 4, 6): ['encipher'], \
 250     #                (5, 0, 6, 1, 3, 4, 2): ['fourteen'], \
 251     #                (6, 1, 0, 4, 5, 3, 2): ['keyword']}) # doctest: +ELLIPSIS
 252     # (((2, 0, 5, 3, 1, 4, 6), False), 0.0628106372...)
 253     # >>> column_transposition_break_mp(column_transposition_encipher(sanitise( \
 254     #         "It is a truth universally acknowledged, that a single man in \
 255     #          possession of a good fortune, must be in want of a wife. However \
 256     #          little known the feelings or views of such a man may be on his \
 257     #          first entering a neighbourhood, this truth is so well fixed in the \
 258     #          minds of the surrounding families, that he is considered the \
 259     #          rightful property of some one or other of their daughters."), \
 260     #     'encipher'), \
 261     #     translist={(2, 0, 5, 3, 1, 4, 6): ['encipher'], \
 262     #                (5, 0, 6, 1, 3, 4, 2): ['fourteen'], \
 263     #                (6, 1, 0, 4, 5, 3, 2): ['keyword']}, \
 264     #     target_counts=normalised_english_trigram_counts) # doctest: +ELLIPSIS
 265     # (((2, 0, 5, 3, 1, 4, 6), False), 0.0592259560...)
 266     # """
 267     ngram_length = len(next(iter(target_counts.keys())))
 268     with Pool() as pool:
 269         helper_args = [(message, trans, columnwise, metric, target_counts, ngram_length,
 270                         message_frequency_scaling)
 271                        for trans in translist.keys() for columnwise in [True, False]]
 272         # Gotcha: the helper function here needs to be defined at the top level
 273         #   (limitation of Pool.starmap)
 274         breaks = pool.starmap(column_transposition_break_worker, helper_args, chunksize)
 275         return min(breaks, key=lambda k: k[1])
 276 column_transposition_break = column_transposition_break_mp
 277
 278 def column_transposition_break_worker(message, transposition, columnwise, metric, target_counts,
 279                       ngram_length, message_frequency_scaling):
 280     plaintext = column_transposition_decipher(message, transposition, columnwise=columnwise)
 281     counts = message_frequency_scaling(frequencies(
 282                          ngrams(sanitise(plaintext), ngram_length)))
 283     fit = metric(target_counts, counts)
 284     logger.debug('Column transposition break attempt using key {0} '
 285                          'gives fit of {1} and decrypt starting: {2}'.format(
 286                              transposition, fit,
 287                              sanitise(plaintext)[:50]))
 288     return (transposition, columnwise), fit
 289
 290
 291 def transposition_break_exhaustive(message):
 292     best_transposition = ''
 293     best_pw = -float('inf')
 294     for keylength in range(1, 21):
 295         if len(message) % keylength == 0:
 296             for transposition in permutations(range(keylength)):
 297                 for columnwise in [True, False]:
 298                     plaintext = column_transposition_decipher(message,
 299                         transposition, columnwise=columnwise)
 300                     # pw = Pwords(segment(plaintext))
 301                     pw = sum([log10(bigram_likelihood(b,
 302                                               normalised_english_bigram_counts,
 303                                               normalised_english_counts))
 304                                            for b in ngrams(plaintext, 2)])
 305                     logger.debug('Column transposition break attempt using key {0} {1} '
 306                          'gives fit of {2} and decrypt starting: {3}'.format(
 307                              transposition, columnwise, pw,
 308                              sanitise(plaintext)[:50]))
 309                     if pw > best_pw:
 310                         best_transposition = transposition
 311                         best_columnwise = columnwise
 312                         best_pw = pw
 313     return (best_transposition, best_columnwise), best_pw
 314
 315
 316 def vigenere_keyword_break(message,
 317                   wordlist=keywords,
 318                   metric=norms.euclidean_distance,
 319                   target_counts=normalised_english_counts,
 320                   message_frequency_scaling=norms.normalise):
 321     """Breaks a vigenere cipher using a dictionary and
 322     frequency analysis
 323
 324     >>> vigenere_keyword_break(vigenere_encipher(sanitise('this is a test ' \
 325              'message for the vigenere decipherment'), 'cat'), \
 326              wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS
 327     ('cat', 0.15965224935...)
 328     """
 329     best_keyword = ''
 330     best_fit = float("inf")
 331     for keyword in wordlist:
 332         plaintext = vigenere_decipher(message, keyword)
 333         counts = message_frequency_scaling(frequencies(plaintext))
 334         fit = metric(target_counts, counts)
 335         logger.debug('Vigenere break attempt using key {0} '
 336                          'gives fit of {1} and decrypt starting: {2}'.format(
 337                              keyword, fit,
 338                              sanitise(plaintext)[:50]))
 339         if fit < best_fit:
 340             best_fit = fit
 341             best_keyword = keyword
 342     logger.info('Vigenere break best fit with key {0} gives fit '
 343                 'of {1} and decrypt starting: {2}'.format(best_keyword,
 344                     best_fit, sanitise(
 345                         vigenere_decipher(message, best_keyword))[:50]))
 346     return best_keyword, best_fit
 347
 348 def vigenere_keyword_break_mp(message,
 349                      wordlist=keywords,
 350                      metric=norms.euclidean_distance,
 351                      target_counts=normalised_english_counts,
 352                      message_frequency_scaling=norms.normalise,
 353                      chunksize=500):
 354     """Breaks a vigenere cipher using a dictionary and
 355     frequency analysis
 356
 357     >>> vigenere_keyword_break_mp(vigenere_encipher(sanitise('this is a test ' \
 358              'message for the vigenere decipherment'), 'cat'), \
 359              wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS
 360     ('cat', 0.159652249358...)
 361     """
 362     with Pool() as pool:
 363         helper_args = [(message, word, metric, target_counts,
 364                         message_frequency_scaling)
 365                        for word in wordlist]
 366         # Gotcha: the helper function here needs to be defined at the top level
 367         #   (limitation of Pool.starmap)
 368         breaks = pool.starmap(vigenere_keyword_break_worker, helper_args, chunksize)
 369         return min(breaks, key=lambda k: k[1])
 370
 371 def vigenere_keyword_break_worker(message, keyword, metric, target_counts,
 372                       message_frequency_scaling):
 373     plaintext = vigenere_decipher(message, keyword)
 374     counts = message_frequency_scaling(frequencies(plaintext))
 375     fit = metric(target_counts, counts)
 376     logger.debug('Vigenere keyword break attempt using key {0} gives fit of '
 377                  '{1} and decrypt starting: {2}'.format(keyword,
 378                      fit, sanitise(plaintext)[:50]))
 379     return keyword, fit
 380
 381
 382
 383 def vigenere_frequency_break(message,
 384                   metric=norms.euclidean_distance,
 385                   target_counts=normalised_english_counts,
 386                   message_frequency_scaling=norms.normalise):
 387     """Breaks a Vigenere cipher with frequency analysis
 388
 389     >>> vigenere_frequency_break(vigenere_encipher(sanitise("It is time to " \
 390             "run. She is ready and so am I. I stole Daniel's pocketbook this " \
 391             "afternoon when he left his jacket hanging on the easel in the " \
 392             "attic."), 'florence')) # doctest: +ELLIPSIS
 393     ('florence', 0.077657073...)
 394     """
 395     best_fit = float("inf")
 396     best_key = ''
 397     sanitised_message = sanitise(message)
 398     for trial_length in range(1, 20):
 399         splits = every_nth(sanitised_message, trial_length)
 400         key = ''.join([chr(caesar_break(s)[0] + ord('a')) for s in splits])
 401         plaintext = vigenere_decipher(sanitised_message, key)
 402         counts = message_frequency_scaling(frequencies(plaintext))
 403         fit = metric(target_counts, counts)
 404         logger.debug('Vigenere key length of {0} ({1}) gives fit of {2}'.
 405                      format(trial_length, key, fit))
 406         if fit < best_fit:
 407             best_fit = fit
 408             best_key = key
 409     logger.info('Vigenere break best fit with key {0} gives fit '
 410                 'of {1} and decrypt starting: {2}'.format(best_key,
 411                     best_fit, sanitise(
 412                         vigenere_decipher(message, best_key))[:50]))
 413     return best_key, best_fit
 414
 415 def beaufort_frequency_break(message,
 416                   metric=norms.euclidean_distance,
 417                   target_counts=normalised_english_counts,
 418                   message_frequency_scaling=norms.normalise):
 419     """Breaks a Beaufort cipher with frequency analysis
 420
 421     >>> beaufort_frequency_break(beaufort_encipher(sanitise("It is time to " \
 422             "run. She is ready and so am I. I stole Daniel's pocketbook this " \
 423             "afternoon when he left his jacket hanging on the easel in the " \
 424             "attic."), 'florence')) # doctest: +ELLIPSIS
 425     ('florence', 0.077657073...)
 426     """
 427     best_fit = float("inf")
 428     best_key = ''
 429     sanitised_message = sanitise(message)
 430     for trial_length in range(1, 20):
 431         splits = every_nth(sanitised_message, trial_length)
 432         key = ''.join([chr(caesar_break(s)[0] + ord('a')) for s in splits])
 433         plaintext = beaufort_decipher(sanitised_message, key)
 434         counts = message_frequency_scaling(frequencies(plaintext))
 435         fit = metric(target_counts, counts)
 436         logger.debug('Beaufort key length of {0} ({1}) gives fit of {2}'.
 437                      format(trial_length, key, fit))
 438         if fit < best_fit:
 439             best_fit = fit
 440             best_key = key
 441     logger.info('Beaufort break best fit with key {0} gives fit '
 442                 'of {1} and decrypt starting: {2}'.format(best_key,
 443                     best_fit, sanitise(
 444                         beaufort_decipher(message, best_key))[:50]))
 445     return best_key, best_fit
 446
 447
 448
 449 def plot_frequency_histogram(freqs, sort_key=None):
 450     x = range(len(freqs.keys()))
 451     y = [freqs[l] for l in sorted(freqs.keys(), key=sort_key)]
 452     f = plt.figure()
 453     ax = f.add_axes([0.1, 0.1, 0.9, 0.9])
 454     ax.bar(x, y, align='center')
 455     ax.set_xticks(x)
 456     ax.set_xticklabels(sorted(freqs.keys(), key=sort_key))
 457     f.show()
 458
 459
 460 if __name__ == "__main__":
 461     import doctest
 462     doctest.testmod()
 463