playfair_develop.ipynb

   1 {
   2  "cells": [
   3   {
   4    "cell_type": "code",
   5    "execution_count": 103,
   6    "metadata": {},
   7    "outputs": [],
   8    "source": [
   9     "import multiprocessing\n",
  10     "from support.utilities import *\n",
  11     "from support.language_models import *\n",
  12     "from support.norms import *\n",
  13     "from cipher.keyword_cipher import *\n",
  14     "from cipher.polybius import *"
  15    ]
  16   },
  17   {
  18    "cell_type": "code",
  19    "execution_count": 23,
  20    "metadata": {},
  21    "outputs": [
  22     {
  23      "data": {
  24       "text/plain": [
  25        "{'p': (1, 1),\n",
  26        " 'l': (1, 2),\n",
  27        " 'a': (1, 3),\n",
  28        " 'y': (1, 4),\n",
  29        " 'f': (1, 5),\n",
  30        " 'i': (2, 1),\n",
  31        " 'r': (2, 2),\n",
  32        " 'e': (2, 3),\n",
  33        " 'x': (2, 4),\n",
  34        " 'm': (2, 5),\n",
  35        " 'b': (3, 1),\n",
  36        " 'c': (3, 2),\n",
  37        " 'd': (3, 3),\n",
  38        " 'g': (3, 4),\n",
  39        " 'h': (3, 5),\n",
  40        " 'k': (4, 1),\n",
  41        " 'n': (4, 2),\n",
  42        " 'o': (4, 3),\n",
  43        " 'q': (4, 4),\n",
  44        " 's': (4, 5),\n",
  45        " 't': (5, 1),\n",
  46        " 'u': (5, 2),\n",
  47        " 'v': (5, 3),\n",
  48        " 'w': (5, 4),\n",
  49        " 'z': (5, 5),\n",
  50        " 'j': (2, 1)}"
  51       ]
  52      },
  53      "execution_count": 23,
  54      "metadata": {},
  55      "output_type": "execute_result"
  56     }
  57    ],
  58    "source": [
  59     "g = polybius_grid('playfair example', [1,2,3,4,5], [1,2,3,4,5])\n",
  60     "g"
  61    ]
  62   },
  63   {
  64    "cell_type": "code",
  65    "execution_count": 14,
  66    "metadata": {},
  67    "outputs": [],
  68    "source": [
  69     "def playfair_wrap(n, lowest, highest):\n",
  70     "    skip = highest - lowest + 1\n",
  71     "    while n > highest or n < lowest:\n",
  72     "        if n > highest:\n",
  73     "            n -= skip\n",
  74     "        if n < lowest:\n",
  75     "            n += skip\n",
  76     "    return n"
  77    ]
  78   },
  79   {
  80    "cell_type": "code",
  81    "execution_count": 20,
  82    "metadata": {},
  83    "outputs": [
  84     {
  85      "data": {
  86       "text/plain": [
  87        "1"
  88       ]
  89      },
  90      "execution_count": 20,
  91      "metadata": {},
  92      "output_type": "execute_result"
  93     }
  94    ],
  95    "source": [
  96     "playfair_wrap(11, 1, 5)"
  97    ]
  98   },
  99   {
 100    "cell_type": "code",
 101    "execution_count": 66,
 102    "metadata": {},
 103    "outputs": [],
 104    "source": [
 105     "def playfair_encipher_bigram(ab, grid, padding_letter='x'):\n",
 106     "    a, b = ab\n",
 107     "    max_row = max(c[0] for c in grid.values())\n",
 108     "    max_col = max(c[1] for c in grid.values())\n",
 109     "    min_row = min(c[0] for c in grid.values())\n",
 110     "    min_col = min(c[1] for c in grid.values())\n",
 111     "    if a == b:\n",
 112     "        b = padding_letter\n",
 113     "    if grid[a][0] == grid[b][0]:  # same row\n",
 114     "        cp = (grid[a][0], playfair_wrap(grid[a][1] + 1, min_col, max_col))\n",
 115     "        dp = (grid[b][0], playfair_wrap(grid[b][1] + 1, min_col, max_col))\n",
 116     "    elif grid[a][1] == grid[b][1]:  # same column\n",
 117     "        cp = (playfair_wrap(grid[a][0] + 1, min_row, max_row), grid[a][1])\n",
 118     "        dp = (playfair_wrap(grid[b][0] + 1, min_row, max_row), grid[b][1])\n",
 119     "    else:\n",
 120     "        cp = (grid[a][0], grid[b][1])\n",
 121     "        dp = (grid[b][0], grid[a][1])\n",
 122     "    c = [k for k, v in grid.items() if v == cp][0]\n",
 123     "    d = [k for k, v in grid.items() if v == dp][0]\n",
 124     "    return c + d"
 125    ]
 126   },
 127   {
 128    "cell_type": "code",
 129    "execution_count": 68,
 130    "metadata": {},
 131    "outputs": [
 132     {
 133      "data": {
 134       "text/plain": [
 135        "'xm'"
 136       ]
 137      },
 138      "execution_count": 68,
 139      "metadata": {},
 140      "output_type": "execute_result"
 141     }
 142    ],
 143    "source": [
 144     "playfair_encipher_bigram('ex', g)"
 145    ]
 146   },
 147   {
 148    "cell_type": "code",
 149    "execution_count": 69,
 150    "metadata": {},
 151    "outputs": [],
 152    "source": [
 153     "def playfair_decipher_bigram(ab, grid, padding_letter='x'):\n",
 154     "    a, b = ab\n",
 155     "    max_row = max(c[0] for c in grid.values())\n",
 156     "    max_col = max(c[1] for c in grid.values())\n",
 157     "    min_row = min(c[0] for c in grid.values())\n",
 158     "    min_col = min(c[1] for c in grid.values())\n",
 159     "    if a == b:\n",
 160     "        b = padding_letter\n",
 161     "    if grid[a][0] == grid[b][0]:  # same row\n",
 162     "        cp = (grid[a][0], playfair_wrap(grid[a][1] - 1, min_col, max_col))\n",
 163     "        dp = (grid[b][0], playfair_wrap(grid[b][1] - 1, min_col, max_col))\n",
 164     "    elif grid[a][1] == grid[b][1]:  # same column\n",
 165     "        cp = (playfair_wrap(grid[a][0] - 1, min_row, max_row), grid[a][1])\n",
 166     "        dp = (playfair_wrap(grid[b][0] - 1, min_row, max_row), grid[b][1])\n",
 167     "    else:\n",
 168     "        cp = (grid[a][0], grid[b][1])\n",
 169     "        dp = (grid[b][0], grid[a][1])\n",
 170     "    c = [k for k, v in grid.items() if v == cp][0]\n",
 171     "    d = [k for k, v in grid.items() if v == dp][0]\n",
 172     "    return c + d"
 173    ]
 174   },
 175   {
 176    "cell_type": "code",
 177    "execution_count": 70,
 178    "metadata": {},
 179    "outputs": [
 180     {
 181      "data": {
 182       "text/plain": [
 183        "'ex'"
 184       ]
 185      },
 186      "execution_count": 70,
 187      "metadata": {},
 188      "output_type": "execute_result"
 189     }
 190    ],
 191    "source": [
 192     "playfair_decipher_bigram('xm', g)"
 193    ]
 194   },
 195   {
 196    "cell_type": "code",
 197    "execution_count": 36,
 198    "metadata": {},
 199    "outputs": [
 200     {
 201      "data": {
 202       "text/plain": [
 203        "['hi', 'de', 'th', 'eg', 'ol', 'di', 'nt', 'he', 'tr', 'ee', 'st', 'um', 'p']"
 204       ]
 205      },
 206      "execution_count": 36,
 207      "metadata": {},
 208      "output_type": "execute_result"
 209     }
 210    ],
 211    "source": [
 212     "chunks(sanitise('hide the gold in the tree stump'), 2)"
 213    ]
 214   },
 215   {
 216    "cell_type": "code",
 217    "execution_count": 75,
 218    "metadata": {},
 219    "outputs": [],
 220    "source": [
 221     "def playfair_bigrams(text, padding_letter='x', padding_replaces_repeat=True):\n",
 222     "    i = 0\n",
 223     "    bigrams = []\n",
 224     "    while i < len(text):\n",
 225     "        bigram = text[i:i+2]\n",
 226     "        if len(bigram) == 1:\n",
 227     "            i = len(text) + 1\n",
 228     "            bigram = bigram + padding_letter\n",
 229     "        else:\n",
 230     "            if bigram[0] == bigram[1]:\n",
 231     "                bigram = bigram[0] + padding_letter\n",
 232     "                if padding_replaces_repeat:\n",
 233     "                    i += 2\n",
 234     "                else:\n",
 235     "                    i += 1\n",
 236     "            else:\n",
 237     "                i += 2\n",
 238     "        bigrams += [bigram]\n",
 239     "    return bigrams\n",
 240     "    "
 241    ]
 242   },
 243   {
 244    "cell_type": "code",
 245    "execution_count": 76,
 246    "metadata": {},
 247    "outputs": [
 248     {
 249      "data": {
 250       "text/plain": [
 251        "['hi', 'de', 'th', 'eg', 'ol', 'di', 'nt', 'he', 'tr', 'ex', 'st', 'um', 'px']"
 252       ]
 253      },
 254      "execution_count": 76,
 255      "metadata": {},
 256      "output_type": "execute_result"
 257     }
 258    ],
 259    "source": [
 260     "playfair_bigrams(sanitise('hide the gold in the tree stump'), padding_replaces_repeat=True)"
 261    ]
 262   },
 263   {
 264    "cell_type": "code",
 265    "execution_count": 77,
 266    "metadata": {},
 267    "outputs": [
 268     {
 269      "data": {
 270       "text/plain": [
 271        "['hi', 'de', 'th', 'eg', 'ol', 'di', 'nt', 'he', 'tr', 'ex', 'es', 'tu', 'mp']"
 272       ]
 273      },
 274      "execution_count": 77,
 275      "metadata": {},
 276      "output_type": "execute_result"
 277     }
 278    ],
 279    "source": [
 280     "playfair_bigrams(sanitise('hide the gold in the tree stump'), padding_replaces_repeat=False)"
 281    ]
 282   },
 283   {
 284    "cell_type": "code",
 285    "execution_count": 73,
 286    "metadata": {},
 287    "outputs": [
 288     {
 289      "data": {
 290       "text/plain": [
 291        "'bmodzbxdnabekudmuixmmouvif'"
 292       ]
 293      },
 294      "execution_count": 73,
 295      "metadata": {},
 296      "output_type": "execute_result"
 297     }
 298    ],
 299    "source": [
 300     "ct = cat(playfair_encipher_bigram((b[0], b[1]), g) for b in playfair_bigrams(sanitise('hide the gold in the tree stump')))\n",
 301     "ct"
 302    ]
 303   },
 304   {
 305    "cell_type": "code",
 306    "execution_count": 74,
 307    "metadata": {},
 308    "outputs": [
 309     {
 310      "data": {
 311       "text/plain": [
 312        "'hidethegoldinthetrexestump'"
 313       ]
 314      },
 315      "execution_count": 74,
 316      "metadata": {},
 317      "output_type": "execute_result"
 318     }
 319    ],
 320    "source": [
 321     "cat(playfair_decipher_bigram((b[0], b[1]), g) for b in playfair_bigrams(sanitise(ct)))"
 322    ]
 323   },
 324   {
 325    "cell_type": "code",
 326    "execution_count": 128,
 327    "metadata": {},
 328    "outputs": [],
 329    "source": [
 330     "def playfair_encipher(message, keyword, padding_letter='x',\n",
 331     "                      padding_replaces_repeat=False,\n",
 332     "#                       column_order=None, row_order=None, \n",
 333     "#                       column_first=False, \n",
 334     "                      letters_to_merge=None, \n",
 335     "                      wrap_alphabet=KeywordWrapAlphabet.from_a):\n",
 336     "    column_order = list(range(5))\n",
 337     "    row_order = list(range(5))\n",
 338     "    if letters_to_merge is None: \n",
 339     "        letters_to_merge = {'j': 'i'}   \n",
 340     "    grid = polybius_grid(keyword, column_order, row_order,\n",
 341     "                        letters_to_merge=letters_to_merge,\n",
 342     "                        wrap_alphabet=wrap_alphabet)\n",
 343     "    message_bigrams = playfair_bigrams(sanitise(message), padding_letter=padding_letter, \n",
 344     "                                       padding_replaces_repeat=padding_replaces_repeat)\n",
 345     "    ciphertext_bigrams = [playfair_encipher_bigram(b, grid, padding_letter=padding_letter) for b in message_bigrams]\n",
 346     "    return cat(ciphertext_bigrams)"
 347    ]
 348   },
 349   {
 350    "cell_type": "code",
 351    "execution_count": 129,
 352    "metadata": {},
 353    "outputs": [],
 354    "source": [
 355     "def playfair_decipher(message, keyword, padding_letter='x',\n",
 356     "                      padding_replaces_repeat=False,\n",
 357     "#                       column_order=None, row_order=None, \n",
 358     "#                       column_first=False, \n",
 359     "                      letters_to_merge=None, \n",
 360     "                      wrap_alphabet=KeywordWrapAlphabet.from_a):\n",
 361     "    column_order = list(range(5))\n",
 362     "    row_order = list(range(5))\n",
 363     "    if letters_to_merge is None: \n",
 364     "        letters_to_merge = {'j': 'i'}   \n",
 365     "    grid = polybius_grid(keyword, column_order, row_order,\n",
 366     "                        letters_to_merge=letters_to_merge,\n",
 367     "                        wrap_alphabet=wrap_alphabet)\n",
 368     "    message_bigrams = playfair_bigrams(sanitise(message), padding_letter=padding_letter, \n",
 369     "                                       padding_replaces_repeat=padding_replaces_repeat)\n",
 370     "    plaintext_bigrams = [playfair_decipher_bigram(b, grid, padding_letter=padding_letter) for b in message_bigrams]\n",
 371     "    return cat(plaintext_bigrams)"
 372    ]
 373   },
 374   {
 375    "cell_type": "code",
 376    "execution_count": 130,
 377    "metadata": {},
 378    "outputs": [
 379     {
 380      "data": {
 381       "text/plain": [
 382        "('bmodzbxdnabekudmuixmkzzryi', 'hidethegoldinthetrexstumpx')"
 383       ]
 384      },
 385      "execution_count": 130,
 386      "metadata": {},
 387      "output_type": "execute_result"
 388     }
 389    ],
 390    "source": [
 391     "prr = True\n",
 392     "plaintext = 'hide the gold in the tree stump'\n",
 393     "key = 'playfair example'\n",
 394     "ciphertext = playfair_encipher(plaintext, key, padding_replaces_repeat=prr)\n",
 395     "recovered_plaintext = playfair_decipher(ciphertext, key, padding_replaces_repeat=prr)\n",
 396     "ciphertext, recovered_plaintext"
 397    ]
 398   },
 399   {
 400    "cell_type": "code",
 401    "execution_count": 131,
 402    "metadata": {},
 403    "outputs": [
 404     {
 405      "data": {
 406       "text/plain": [
 407        "('bmodzbxdnabekudmuixmmouvif', 'hidethegoldinthetrexestump')"
 408       ]
 409      },
 410      "execution_count": 131,
 411      "metadata": {},
 412      "output_type": "execute_result"
 413     }
 414    ],
 415    "source": [
 416     "prr = False\n",
 417     "plaintext = 'hide the gold in the tree stump'\n",
 418     "key = 'playfair example'\n",
 419     "ciphertext = playfair_encipher(plaintext, key, padding_replaces_repeat=prr)\n",
 420     "recovered_plaintext = playfair_decipher(ciphertext, key, padding_replaces_repeat=prr)\n",
 421     "ciphertext, recovered_plaintext"
 422    ]
 423   },
 424   {
 425    "cell_type": "code",
 426    "execution_count": 132,
 427    "metadata": {},
 428    "outputs": [
 429     {
 430      "data": {
 431       "text/plain": [
 432        "('dlckztactiokoncbntaucenzpl', 'hidethegoldinthetrexestump')"
 433       ]
 434      },
 435      "execution_count": 132,
 436      "metadata": {},
 437      "output_type": "execute_result"
 438     }
 439    ],
 440    "source": [
 441     "prr = False\n",
 442     "plaintext = 'hide the gold in the tree stump'\n",
 443     "key = 'simple key'\n",
 444     "ciphertext = playfair_encipher(plaintext, key, padding_replaces_repeat=prr)\n",
 445     "recovered_plaintext = playfair_decipher(ciphertext, key, padding_replaces_repeat=prr)\n",
 446     "ciphertext, recovered_plaintext"
 447    ]
 448   },
 449   {
 450    "cell_type": "code",
 451    "execution_count": 134,
 452    "metadata": {},
 453    "outputs": [],
 454    "source": [
 455     "def playfair_break_mp(message, \n",
 456     "                      letters_to_merge=None, padding_letter='x',\n",
 457     "                      wordlist=keywords, fitness=Pletters,\n",
 458     "                      number_of_solutions=1, chunksize=500):\n",
 459     "    if letters_to_merge is None: \n",
 460     "        letters_to_merge = {'j': 'i'}   \n",
 461     "\n",
 462     "    with multiprocessing.Pool() as pool:\n",
 463     "        helper_args = [(message, word, wrap, \n",
 464     "                        letters_to_merge, padding_letter,\n",
 465     "                        pad_replace,\n",
 466     "                        fitness)\n",
 467     "                       for word in wordlist\n",
 468     "                       for wrap in KeywordWrapAlphabet\n",
 469     "                       for pad_replace in [False, True]]\n",
 470     "        # Gotcha: the helper function here needs to be defined at the top level\n",
 471     "        #   (limitation of Pool.starmap)\n",
 472     "        breaks = pool.starmap(playfair_break_worker, helper_args, chunksize)\n",
 473     "        if number_of_solutions == 1:\n",
 474     "            return max(breaks, key=lambda k: k[1])\n",
 475     "        else:\n",
 476     "            return sorted(breaks, key=lambda k: k[1], reverse=True)[:number_of_solutions]"
 477    ]
 478   },
 479   {
 480    "cell_type": "code",
 481    "execution_count": 135,
 482    "metadata": {},
 483    "outputs": [],
 484    "source": [
 485     "def playfair_break_worker(message, keyword, wrap, \n",
 486     "                          letters_to_merge, padding_letter,\n",
 487     "                          pad_replace,\n",
 488     "                          fitness):\n",
 489     "    plaintext = playfair_decipher(message, keyword, padding_letter,\n",
 490     "                                  pad_replace,\n",
 491     "                                  letters_to_merge, \n",
 492     "                                  wrap)\n",
 493     "    if plaintext:\n",
 494     "        fit = fitness(plaintext)\n",
 495     "    else:\n",
 496     "        fit = float('-inf')\n",
 497     "    logger.debug('Playfair break attempt using key {0} (wrap={1}, merging {2}, '\n",
 498     "                 'pad replaces={3}), '\n",
 499     "                 'gives fit of {4} and decrypt starting: '\n",
 500     "                 '{5}'.format(keyword, wrap, letters_to_merge, pad_replace,\n",
 501     "                              fit, sanitise(plaintext)[:50]))\n",
 502     "    return (keyword, wrap, letters_to_merge, padding_letter, pad_replace), fit"
 503    ]
 504   },
 505   {
 506    "cell_type": "code",
 507    "execution_count": 136,
 508    "metadata": {},
 509    "outputs": [
 510     {
 511      "data": {
 512       "text/plain": [
 513        "(('elephant', <KeywordWrapAlphabet.from_a: 1>, {'j': 'i'}, 'x', False),\n",
 514        " -54.53880323982303)"
 515       ]
 516      },
 517      "execution_count": 136,
 518      "metadata": {},
 519      "output_type": "execute_result"
 520     }
 521    ],
 522    "source": [
 523     "playfair_break_mp(playfair_encipher('this is a test message for the ' \\\n",
 524     "          'polybius decipherment', 'elephant'), \\\n",
 525     "          wordlist=['cat', 'elephant', 'kangaroo']) "
 526    ]
 527   },
 528   {
 529    "cell_type": "code",
 530    "execution_count": 200,
 531    "metadata": {},
 532    "outputs": [],
 533    "source": [
 534     "def playfair_simulated_annealing_break(message, workers=10, \n",
 535     "                              initial_temperature=200,\n",
 536     "                              max_iterations=20000,\n",
 537     "                              plain_alphabet=None, \n",
 538     "                              cipher_alphabet=None, \n",
 539     "                              fitness=Pletters, chunksize=1):\n",
 540     "    worker_args = []\n",
 541     "    ciphertext = sanitise(message)\n",
 542     "    for i in range(workers):\n",
 543     "        if plain_alphabet is None:\n",
 544     "            used_plain_alphabet = string.ascii_lowercase\n",
 545     "        else:\n",
 546     "            used_plain_alphabet = plain_alphabet\n",
 547     "        if cipher_alphabet is None:\n",
 548     "#             used_cipher_alphabet = list(string.ascii_lowercase)\n",
 549     "#             random.shuffle(used_cipher_alphabet)\n",
 550     "#             used_cipher_alphabet = cat(used_cipher_alphabet)\n",
 551     "            used_cipher_alphabet = random.choice(keywords)\n",
 552     "        else:\n",
 553     "            used_cipher_alphabet = cipher_alphabet\n",
 554     "        worker_args.append((ciphertext, used_plain_alphabet, used_cipher_alphabet, \n",
 555     "                            initial_temperature, max_iterations, fitness))\n",
 556     "    with multiprocessing.Pool() as pool:\n",
 557     "        breaks = pool.starmap(playfair_simulated_annealing_break_worker,\n",
 558     "                              worker_args, chunksize)\n",
 559     "    return max(breaks, key=lambda k: k[1])"
 560    ]
 561   },
 562   {
 563    "cell_type": "code",
 564    "execution_count": 205,
 565    "metadata": {},
 566    "outputs": [],
 567    "source": [
 568     "def playfair_simulated_annealing_break_worker(message, plain_alphabet, cipher_alphabet, \n",
 569     "                                     t0, max_iterations, fitness):\n",
 570     "    def swap(letters, i, j):\n",
 571     "        if i > j:\n",
 572     "            i, j = j, i\n",
 573     "        if i == j:\n",
 574     "            return letters\n",
 575     "        else:\n",
 576     "            return (letters[:i] + letters[j] + letters[i+1:j] + letters[i] +\n",
 577     "                    letters[j+1:])\n",
 578     "    \n",
 579     "    temperature = t0\n",
 580     "\n",
 581     "    dt = t0 / (0.9 * max_iterations)\n",
 582     "    \n",
 583     "    current_alphabet = cipher_alphabet\n",
 584     "#     current_wrap = KeywordWrapAlphabet.from_a\n",
 585     "    current_letters_to_merge = {'j': 'i'}\n",
 586     "    current_pad_replace = False\n",
 587     "    current_padding_letter = 'x'\n",
 588     "    \n",
 589     "    alphabet = current_alphabet\n",
 590     "#     wrap = current_wrap\n",
 591     "    letters_to_merge = current_letters_to_merge\n",
 592     "    pad_replace = current_pad_replace\n",
 593     "    padding_letter = current_padding_letter\n",
 594     "    plaintext = playfair_decipher(message, alphabet, padding_letter,\n",
 595     "                                  pad_replace,\n",
 596     "                                  letters_to_merge, \n",
 597     "                                  KeywordWrapAlphabet.from_a)\n",
 598     "    current_fitness = fitness(plaintext)\n",
 599     "\n",
 600     "    best_alphabet = current_alphabet\n",
 601     "#     best_wrap = current_wrap\n",
 602     "    best_letters_to_merge = current_letters_to_merge\n",
 603     "    best_pad_replace = current_pad_replace\n",
 604     "    best_padding_letter = current_padding_letter\n",
 605     "    best_fitness = current_fitness\n",
 606     "    best_plaintext = plaintext\n",
 607     "    \n",
 608     "    # print('starting for', max_iterations)\n",
 609     "    for i in range(max_iterations):\n",
 610     "        chosen = random.random()\n",
 611     "#         if chosen < 0.7:\n",
 612     "#             swap_a = random.randrange(26)\n",
 613     "#             swap_b = (swap_a + int(random.gauss(0, 4))) % 26\n",
 614     "#             alphabet = swap(current_alphabet, swap_a, swap_b)\n",
 615     "# #         elif chosen < 0.8:\n",
 616     "# #             wrap = random.choice(list(KeywordWrapAlphabet))\n",
 617     "#         elif chosen < 0.8:\n",
 618     "#             pad_replace = random.choice([True, False])\n",
 619     "#         elif chosen < 0.9:\n",
 620     "#             letter_from = random.choice(string.ascii_lowercase)\n",
 621     "#             letter_to = random.choice([c for c in string.ascii_lowercase if c != letter_from])\n",
 622     "#             letters_to_merge = {letter_from: letter_to}\n",
 623     "#         else:\n",
 624     "#             padding_letter = random.choice(string.ascii_lowercase)\n",
 625     "\n",
 626     "        if chosen < 0.7:\n",
 627     "            swap_a = random.randrange(len(current_alphabet))\n",
 628     "            swap_b = (swap_a + int(random.gauss(0, 4))) % len(current_alphabet)\n",
 629     "            alphabet = swap(current_alphabet, swap_a, swap_b)\n",
 630     "        elif chosen < 0.85:\n",
 631     "            new_letter = random.choice(string.ascii_lowercase)\n",
 632     "            alphabet = swap(current_alphabet + new_letter, random.randrange(len(current_alphabet)), len(current_alphabet))\n",
 633     "        else:\n",
 634     "            if len(current_alphabet) > 1:\n",
 635     "                deletion_position = random.randrange(len(current_alphabet))\n",
 636     "                alphabet = current_alphabet[:deletion_position] + current_alphabet[deletion_position+1:]\n",
 637     "            else:\n",
 638     "                alphabet = current_alphabet\n",
 639     "\n",
 640     "        try:\n",
 641     "            plaintext = playfair_decipher(message, alphabet, padding_letter,\n",
 642     "                                  pad_replace,\n",
 643     "                                  letters_to_merge, \n",
 644     "                                  KeywordWrapAlphabet.from_a)\n",
 645     "        except:\n",
 646     "            print(\"Error\", alphabet, padding_letter,\n",
 647     "                                  pad_replace,\n",
 648     "                                  letters_to_merge)\n",
 649     "            raise\n",
 650     "\n",
 651     "        new_fitness = fitness(plaintext)\n",
 652     "        try:\n",
 653     "            sa_chance = math.exp((new_fitness - current_fitness) / temperature)\n",
 654     "        except (OverflowError, ZeroDivisionError):\n",
 655     "            # print('exception triggered: new_fit {}, current_fit {}, temp {}'.format(new_fitness, current_fitness, temperature))\n",
 656     "            sa_chance = 0\n",
 657     "        if (new_fitness > current_fitness or random.random() < sa_chance):\n",
 658     "            # logger.debug('Simulated annealing: iteration {}, temperature {}, '\n",
 659     "            #     'current alphabet {}, current_fitness {}, '\n",
 660     "            #     'best_plaintext {}'.format(i, temperature, current_alphabet, \n",
 661     "            #     current_fitness, best_plaintext[:50]))\n",
 662     "\n",
 663     "            # logger.debug('new_fit {}, current_fit {}, temp {}, sa_chance {}'.format(new_fitness, current_fitness, temperature, sa_chance))\n",
 664     "            current_fitness = new_fitness\n",
 665     "            current_alphabet = alphabet\n",
 666     "#             current_wrap = wrap\n",
 667     "            current_letters_to_merge = letters_to_merge\n",
 668     "            current_pad_replace = pad_replace\n",
 669     "            current_padding_letter = padding_letter\n",
 670     "            \n",
 671     "        if current_fitness > best_fitness:\n",
 672     "            best_alphabet = current_alphabet\n",
 673     "#             best_wrap = current_wrap\n",
 674     "            best_letters_to_merge = current_letters_to_merge\n",
 675     "            best_pad_replace = current_pad_replace\n",
 676     "            best_padding_letter = current_padding_letter\n",
 677     "            best_fitness = current_fitness\n",
 678     "            best_plaintext = plaintext\n",
 679     "        if i % 500 == 0:\n",
 680     "            logger.debug('Simulated annealing: iteration {}, temperature {}, '\n",
 681     "                'current alphabet {}, current_fitness {}, '\n",
 682     "                'best_plaintext {}'.format(i, temperature, current_alphabet, \n",
 683     "                current_fitness, plaintext[:50]))\n",
 684     "        temperature = max(temperature - dt, 0.001)\n",
 685     "\n",
 686     "    print(best_alphabet, best_plaintext[:50])\n",
 687     "    return { 'alphabet': best_alphabet\n",
 688     "#            , 'wrap': best_wrap\n",
 689     "           , 'letters_to_merge': best_letters_to_merge\n",
 690     "           , 'pad_replace': best_pad_replace\n",
 691     "           , 'padding_letter': best_padding_letter\n",
 692     "           }, best_fitness # current_alphabet, current_fitness"
 693    ]
 694   },
 695   {
 696    "cell_type": "code",
 697    "execution_count": 149,
 698    "metadata": {},
 699    "outputs": [
 700     {
 701      "data": {
 702       "text/plain": [
 703        "'dlckztactiokoncbntaucenzpl'"
 704       ]
 705      },
 706      "execution_count": 149,
 707      "metadata": {},
 708      "output_type": "execute_result"
 709     }
 710    ],
 711    "source": [
 712     "ciphertext"
 713    ]
 714   },
 715   {
 716    "cell_type": "code",
 717    "execution_count": 168,
 718    "metadata": {},
 719    "outputs": [
 720     {
 721      "data": {
 722       "text/plain": [
 723        "({'alphabet': 'ipknagqxvszjrmhlfyeutwdcbo',\n",
 724        "  'wrap': <KeywordWrapAlphabet.from_largest: 3>,\n",
 725        "  'letters_to_merge': {'x': 'z'},\n",
 726        "  'pad_replace': False,\n",
 727        "  'padding_letter': 't'},\n",
 728        " -85.75243058399522)"
 729       ]
 730      },
 731      "execution_count": 168,
 732      "metadata": {},
 733      "output_type": "execute_result"
 734     }
 735    ],
 736    "source": [
 737     "key, score = playfair_simulated_annealing_break(ciphertext, fitness=Ptrigrams)\n",
 738     "key, score"
 739    ]
 740   },
 741   {
 742    "cell_type": "code",
 743    "execution_count": 169,
 744    "metadata": {},
 745    "outputs": [],
 746    "source": [
 747     "# polybius_grid(key['alphabet'], [1,2,3,4,5], [1,2,3,4,5], letters_to_merge=key['letters_to_merge'], wrap_alphabet=key['wrap'])"
 748    ]
 749   },
 750   {
 751    "cell_type": "code",
 752    "execution_count": 170,
 753    "metadata": {},
 754    "outputs": [
 755     {
 756      "data": {
 757       "text/plain": [
 758        "'orecalkofacabadcauntemasar'"
 759       ]
 760      },
 761      "execution_count": 170,
 762      "metadata": {},
 763      "output_type": "execute_result"
 764     }
 765    ],
 766    "source": [
 767     "playfair_decipher(ciphertext, key['alphabet'], key['padding_letter'], key['pad_replace'], key['letters_to_merge'], key['wrap'])"
 768    ]
 769   },
 770   {
 771    "cell_type": "code",
 772    "execution_count": 171,
 773    "metadata": {},
 774    "outputs": [
 775     {
 776      "data": {
 777       "text/plain": [
 778        "('gmearkafusalkufbutbvfeuopl', 'hidethegoldinthetrexestump')"
 779       ]
 780      },
 781      "execution_count": 171,
 782      "metadata": {},
 783      "output_type": "execute_result"
 784     }
 785    ],
 786    "source": [
 787     "prr = False\n",
 788     "plaintext = 'hide the gold in the tree stump'\n",
 789     "key = 'simple'\n",
 790     "ciphertext = playfair_encipher(plaintext, key, padding_replaces_repeat=prr)\n",
 791     "recovered_plaintext = playfair_decipher(ciphertext, key, padding_replaces_repeat=prr)\n",
 792     "ciphertext, recovered_plaintext"
 793    ]
 794   },
 795   {
 796    "cell_type": "code",
 797    "execution_count": 174,
 798    "metadata": {},
 799    "outputs": [
 800     {
 801      "data": {
 802       "text/plain": [
 803        "(('simple', <KeywordWrapAlphabet.from_a: 1>, {'j': 'i'}, 'x', False),\n",
 804        " -80.29349856508469)"
 805       ]
 806      },
 807      "execution_count": 174,
 808      "metadata": {},
 809      "output_type": "execute_result"
 810     }
 811    ],
 812    "source": [
 813     "playfair_break_mp(ciphertext, fitness=Ptrigrams)"
 814    ]
 815   },
 816   {
 817    "cell_type": "code",
 818    "execution_count": 175,
 819    "metadata": {},
 820    "outputs": [
 821     {
 822      "data": {
 823       "text/plain": [
 824        "'hidethegoldinthetrexestump'"
 825       ]
 826      },
 827      "execution_count": 175,
 828      "metadata": {},
 829      "output_type": "execute_result"
 830     }
 831    ],
 832    "source": [
 833     "playfair_decipher(ciphertext, 'simple')"
 834    ]
 835   },
 836   {
 837    "cell_type": "code",
 838    "execution_count": 179,
 839    "metadata": {},
 840    "outputs": [
 841     {
 842      "data": {
 843       "text/plain": [
 844        "({'alphabet': 'rhbylupiwzevjdxfakcqtnomgs',\n",
 845        "  'wrap': <KeywordWrapAlphabet.from_a: 1>,\n",
 846        "  'letters_to_merge': {'p': 'f'},\n",
 847        "  'pad_replace': True,\n",
 848        "  'padding_letter': 'a'},\n",
 849        " -78.01490096572304)"
 850       ]
 851      },
 852      "execution_count": 179,
 853      "metadata": {},
 854      "output_type": "execute_result"
 855     }
 856    ],
 857    "source": [
 858     "key, score = playfair_simulated_annealing_break(ciphertext, fitness=Ptrigrams, workers=50, max_iterations=int(1e5))\n",
 859     "key, score"
 860    ]
 861   },
 862   {
 863    "cell_type": "code",
 864    "execution_count": 180,
 865    "metadata": {},
 866    "outputs": [
 867     {
 868      "data": {
 869       "text/plain": [
 870        "'mouthatventraidleardelines'"
 871       ]
 872      },
 873      "execution_count": 180,
 874      "metadata": {},
 875      "output_type": "execute_result"
 876     }
 877    ],
 878    "source": [
 879     "playfair_decipher(ciphertext, key['alphabet'], key['padding_letter'], key['pad_replace'], key['letters_to_merge'], key['wrap'])"
 880    ]
 881   },
 882   {
 883    "cell_type": "code",
 884    "execution_count": 184,
 885    "metadata": {},
 886    "outputs": [
 887     {
 888      "data": {
 889       "text/plain": [
 890        "'the april uprising in bulgaria and its brutal suppression by the turks has caused outrage in the\\nchancelleries of europe there is a risk that russia will take this as the excuse it seeks to engage\\nthe ottomans and if they act and take constantinople then our trading routes to india will be under\\nthreatat home gladstones pamphlet bulgarian horrors and the question of the east has stirred a\\npublic appetite for action which could lead to support for intervention and make things difficult\\nfor the prime minister he is faced with mortons fork if he supports action then it will be difficult\\nto condemn russian interference if he counsels inaction then he risk appearing weak and callous at\\nhome and abroad it may appear unfortunate that our political leaders are unable to agree on policy\\nstrategy or tactics and it is true that this could lead to confusion about our aims but on\\nreflection i think that the public disagreement between gladstone and disraeli presents an\\nopportunity their dispute conducted in parliament and the press demonstrates to the world the two\\nfaces of the empire at the same time morally engaged and yet prudent this may allow us to proceed\\nwith discretion to try to influence the actors and to direct the play away from the glare of the\\nfootlights it may be possible to engage the league of the three emperors to our causebismarck is\\nparticularly keen to maintain a balance of power in the region and to avoid further war and he will\\nnot need to be convinced that an unbridled russia is not to his advantage so i think we can rely on\\nhim to rein in russias expansionary visionon the other hand the league itself may present a longer\\nterm threat to the empire given the breadth of its influence in northern europe and we must tread\\ncarefullythe emperors envoys will be meeting in reichstadt soon to determine the response to the\\ncrisis and i need a plan to influence the outcome as always our strategy must be to sow confusion\\nand on this i plan to ask for advice from baron playfair he has recently concluded his commission of\\nenquiry into the civil service and if anyone knows how to control an agenda it must be our own civil\\nservants'"
 891       ]
 892      },
 893      "execution_count": 184,
 894      "metadata": {},
 895      "output_type": "execute_result"
 896     }
 897    ],
 898    "source": [
 899     "plaintext = open('2018/5b.plaintext').read()\n",
 900     "plaintext"
 901    ]
 902   },
 903   {
 904    "cell_type": "code",
 905    "execution_count": 186,
 906    "metadata": {},
 907    "outputs": [
 908     {
 909      "data": {
 910       "text/plain": [
 911        "'beitnicknqecarqsrpmzqlsiakspkratshobgkbnxinirtarpogzbetfnhdaibgrbptrfnobistcrpbeihibqrcffcecrtwohoenoibeieictgecadbegahnavartxckfgkptfrctgtariiwhqtreatriftawtqsgbtfriwffwkbvdspkrofrixgegspfskpihpotaspaeopqktfriopnhseskrpscpnfttapevnakxekymghovniebeeigagaeufhlqsktaportxkkucmtfmzqlsiakurneensdspfsriunrtaepowobeiwittaibavtaceeiksqngmchkxoiaeftowisegepovrchreqqmfmitfsntnqqpesowecosiewroseppsvnkbfiberpbtkrkwkehqfgowesrinihkhfrprafterictdgirfxebefuespotdnepamertnqqpestgegeposriprfeckmgrfekkehqfgfweqvnhfvsnbarsprpftedierohiekrieqnotrdgrpgiaepoberoriecadkxoisirptyitpkvnigkyfqnbgaeufhspksshptkrbfgxkxoisinoowesnogatfibfwnhqpkcaeigkyfcskietgeinogsfcfwgbeitwoqqfchvgsegactwqesgiaergspkraetahntfibawberaeqqmfmitfsqepomoarpogspnfwnhkadbmzfwvstofcegepprberpfaibawbeiozmkcrlragbeihfroastfetrolqsktapoitvnkrdstikcnirtroatsppqqpesnoeawgricekranobihpomnegrfrpxkcdakfhosspfsrinirtdnhfpotaisfttawfriewcdfsrifewogirtwobeiwhfxaeigabertbktfhkhfnegkqcrobgtcksvnwcaohnfrosberakbxgkyfqzotapqenhirfxebekrgreiaepofwsewgpeodmqrohibeitegnetgvnfwkreiegbeiokgxgxtwlenfbrilqsitwofriowwfkcbcateakbzgiontargmtfwtqsgbtfrifcgbohwobetfreiwiwhfoiensdfwpnehbptrahbdsiilraxkeschqmsiqcfirofwkbrpagrpgsgksphiwoqpetecosrieiacpospfsptwnrkmoesrievsispmrteckdqwforrffwtheqvrrphifsibagonusecfmrfhnavarkadvwffwrctgnrspagtctnearcpdetigvscfwqurhkfweirprphnavargtiwxkvdeppscvxrarpopobetwbeieibvnbecfitbqicatcfkdgxnirtroagfqqsiefthdbeeigafwbeiwhfxaeiacwrosrishitfseukeatrpkmohqricpvopesrisvhoenoispfvhfnbawseitlrsitoqmqcbeiwhfoiensdrownvgxekymghfiwtfrpacvsichrtaskatpwpofwfrfthdrptfrieianpotrfwbeihecartgvnrpiwrfkxkgospeodmqrohibetwobeqfhgtgkxtvgpnsdsegactzlnbastfntweeqodnbeppsvnpoberaaxkgosptadowisnrchtoenumsipoqkgxktceriibdsihrogfcgpogqnmrfrcgrufkhavarpoworouncexcoswfrihcxrdgiexrhispkrktqvpoifopvteuefqeposeqfspgbrokseauztathpnenvohcxrdgiexsosav'"
 912       ]
 913      },
 914      "execution_count": 186,
 915      "metadata": {},
 916      "output_type": "execute_result"
 917     }
 918    ],
 919    "source": [
 920     "ciphertext = playfair_encipher(plaintext, 'reichstag')\n",
 921     "ciphertext"
 922    ]
 923   },
 924   {
 925    "cell_type": "code",
 926    "execution_count": 189,
 927    "metadata": {},
 928    "outputs": [
 929     {
 930      "data": {
 931       "text/plain": [
 932        "'theapriluprisinginbulgariaanditsbrutalsupxpressionbytheturkshascausedoutrageinthechancelleriesofeuropethereisariskthatrusxsiawilltakethisastheexcuseitseekstoengagetheottomansandiftheyactandtakeconstantinoplethenourtradingroutestoindiawilxlbeunderthreatathomegladstonespamphletbulgarianhorrorsandthequestionofthexeasthasxstirredapublicappetiteforactionwhichcouldleadtosupportforinterventionandmakethingsdifxficultfortheprimeministerheisfacedwithmortonsforkifhesupportsactionthenitwillbedifficulttocondemnrussianinterferenceifhecounselsinactionthenheriskappearingweakandcalxlousathomeandabroaditmayappearunfortunatethatourpoliticalxleadersareunabletoagreeonpolicystrategyortacticsanditistruethatxthiscouldleadtoconfusionaboutouraimsbutonreflectionithinkthatxthepublicdisagreementbetweengladstoneanddisraelipresentsanopportunitytheirdisputeconductedinparliamentandthepressdemonstratestotheworldthetwofacesofthexempireatthesametimemorallyengagedandyetprudentthismayalxlowustoproceedwithdiscretiontotrytoinfluencetheactorsandtodirecttheplayawayfromtheglareofthefootlightsitmaybepossibletoengagetheleagueofthethrexexemperorstoourcausebismarckisparticularlykeentomaintainabalanceofpowerintheregionandtoavoidfurtherwarandhewillnotneedtobeconvincedthatanunbridledrusxsiaisnottohisadvantagesoithinkwecanrelyonhimtoreininrusxsiasexpansionaryvisionontheotherhandtheleagueitselfmaypresentalongertermthreattothexempiregiventhebreadthofitsinfluenceinxnortherneuropeandwemustxtreadcarefullythexemperorsenvoyswilxlbemexetinginreichstadtsoxontodeterminetheresponsetothecrisisandinexedaplantoinfluencetheoutcomeasalwaysourstrategymustbetosowconfusionandonthisiplantoaskforadvicefrombaronplayfairhehasrecentlyconcludedhiscommisxsionofenquiryintothecivilserviceandifanyoneknowshowtocontrolanagendaitmustbeourowncivilservantsx'"
 933       ]
 934      },
 935      "execution_count": 189,
 936      "metadata": {},
 937      "output_type": "execute_result"
 938     }
 939    ],
 940    "source": [
 941     "playfair_decipher(ciphertext, 'reichstag')"
 942    ]
 943   },
 944   {
 945    "cell_type": "code",
 946    "execution_count": 206,
 947    "metadata": {},
 948    "outputs": [
 949     {
 950      "name": "stdout",
 951      "output_type": "stream",
 952      "text": [
 953       "seidfrts tdeapsamupfttcndanculbfeiaingatregqtmhrqpxpscrtcon\n",
 954       "napqbmkvyzgocxuseiftadkianrs atefpsovbpiorenfhazylfterainaebeiduaomqbcfpswereac\n",
 955       "qyobxhweskuzpnvmlfgdapxfechdsnvzhbmyubuhiqmnj oscrprtwubwitibhinluymerthensttrekyodsrsotprirtiei\n",
 956       "reicbstaigk creaprbguprisinginkulgartgansbtsadueagrupxpression\n",
 957       "reichstarg theapriluprisinginbulgariaanditsbrutalsupxpression\n",
 958       "reichstaig theapriluprisinginbulgariaanditsbrutalsupxpression\n",
 959       "reifhst tfeapramuphtsinbinculbariaanditscrqtdgsqpxpression\n",
 960       "stagbreicech theapriluprisinginhulcarxianditsbrutalsupapression\n",
 961       "retarhsi itbeosblupaithnctndulcdstlbnfttedrpifgspwboshethon\n",
 962       "reichstag theapriluprisinginbulgariaanditsbrutalsupxpression\n"
 963      ]
 964     },
 965     {
 966      "data": {
 967       "text/plain": [
 968        "({'alphabet': 'reichstaig',\n",
 969        "  'letters_to_merge': {'j': 'i'},\n",
 970        "  'pad_replace': False,\n",
 971        "  'padding_letter': 'x'},\n",
 972        " -6173.17111571937)"
 973       ]
 974      },
 975      "execution_count": 206,
 976      "metadata": {},
 977      "output_type": "execute_result"
 978     }
 979    ],
 980    "source": [
 981     "key, score = playfair_simulated_annealing_break(ciphertext, fitness=Ptrigrams, workers=10, max_iterations=int(1e5), initial_temperature=500)\n",
 982     "key, score"
 983    ]
 984   },
 985   {
 986    "cell_type": "code",
 987    "execution_count": null,
 988    "metadata": {},
 989    "outputs": [],
 990    "source": []
 991   }
 992  ],
 993  "metadata": {
 994   "kernelspec": {
 995    "display_name": "Python 3",
 996    "language": "python",
 997    "name": "python3"
 998   },
 999   "language_info": {
1000    "codemirror_mode": {
1001     "name": "ipython",
1002     "version": 3
1003    },
1004    "file_extension": ".py",
1005    "mimetype": "text/x-python",
1006    "name": "python",
1007    "nbconvert_exporter": "python",
1008    "pygments_lexer": "ipython3",
1009    "version": "3.6.6"
1010   }
1011  },
1012  "nbformat": 4,
1013  "nbformat_minor": 2
1014 }