Implemented polybius ciphers, updated challenge 3
[cipher-tools.git] / 2017 / polybius-development.ipynb
diff --git a/2017/polybius-development.ipynb b/2017/polybius-development.ipynb
new file mode 100644 (file)
index 0000000..eb81437
--- /dev/null
@@ -0,0 +1,1086 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os,sys,inspect, collections\n",
+    "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
+    "parentdir = os.path.dirname(currentdir)\n",
+    "sys.path.insert(0,parentdir) \n",
+    "\n",
+    "import matplotlib.pyplot as plt\n",
+    "%matplotlib inline\n",
+    "\n",
+    "from cipherbreak import *\n",
+    "\n",
+    "ca = open('3a.ciphertext').read()\n",
+    "cb = open('3b.ciphertext').read()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'MHALEATASTIFETHOTSFETNRAFSSETFWTHEYAROLPOTTLEMATHPNFDAUUOOTMOTLARGSTIEETHEHODREATHEITHETINNWSRNITHETOUTAUOLOPALATBTNNVEIUNCETHEHFGERFCEIAUOLDASODVORTOGEYOUEDPBHASOICBMATHNFTTHESFWWNITNYOGIAUNLOORDHASTOLERTEDSUNFTSDEYEOTMNFLDHOVEPEERAREVATOPLEORDMATHATTHELNSSNYPIATORRAOHOVARGIEOLASEDTHOTTHEAUERAMEIEIEODARGHASUNCCFRAUOTANRSOGIAUNLOOIIORGEDTNWLORTYOLSEARYNICOTANROPNFTTHEWLORREDDASWNSATANRNYTINNWSSETTARGOTIOWARTNMHAUHTHEAUERAYELLATASSTALLFRULEOIHNMTHESOVOGETIAPESHODLEOIREDTNDEUAWHEINFIOICBSUNCCFRAUOTANRSOSTHEBHODRNMIATARGNYTHEAINMRHNMEVEIEVEIBOICBHOSATSTIOATNISORDATSEECSLAKELBTNCETHOTNRENICNIENYTHELNUOLTIAPESWENWLETFIREDSNCENRESHEODLNVENICNREBATASOLLTIEOUHEIBPFTTHASOUTTHIEOTEREDTNFRDEICARETHEECWAIESERTAIESBSTECNYSEUFIEUNCCFRAUOTANRTHEUOESOIUAWHEIWEIHOWSCNIETHORINODSHODEROPLEDTHEECWAIETNYFRUTANRSEUFIELBORDATSLNSSMOSODASOSTEITFIRARGTHOTLNSSARTNOVAUTNIBNRTHEPOTTLEYAELDDESEIVEDCNIEIEUNGRATANRTHORATIEUEAVEDWEIHOWSSFETNRAFSREVEIKREMNIWEIHOWSHEYOALEDTNFRDEISTORDTHEUFRRARGNYTHEWLORPFTEATHEIMOBDESWATETHEVAUTNIBOGIAUNLOORDTHERARTHLEGANRIECOAREDARDASGIOUENRHEOIARGTHASREMSULOFDAFSUOESOIOFGFSTFSGEICORAUFSREINECWEINIASSFEDOSEUIETWINULOCOTANRESTOPLASHARGOUAWHEISUHNNLTNDEVELNWOREMSBSTECNYACWEIAOLUAWHEISATMNIKEDARSEUIETYNINVEITMERTBBEOISDEVELNWARGREMCETHNDSTNSEUFIEUNCCFRAUOTANRSOUINSSTHEECWAIESTFDBARGTHEMNIKSNYTHEGIEEKSOGESEFULADORDHBWOTAOORDEVERNLDEIMNIKSYINCORUAERTPOPBLNRTHESUHNNLDEVELNWEDREMMOBSTNSOYEGFOIDUNCCFRAUOTANRSYINCINCESERECAESTHESEREMSBSTECSMEIEMIATTERDNMRORDDASTIAPFTEDARTHEUNDEXNUUFLTNIFCWEIHOWSTHECNSTSTIAUTLBGFOIDEDDNUFCERTAROLLHASTNIBMHERATMOSYAROLLBUNCWLETEDCORBBEOISLOTEITHEECWEINIDNCATAORASSFEDOREXEUFTAVENIDEIMHAUHWIEYOUEDEVEIBUNWBNYTHEUNDEXTHOTATMOSTNPEGFOIDEDPBEVEIBLEGANRTNTHELOSTCORTHEDEYEOTNYTHEAUERAORDTHESFAUADENYPNFDAUUOWEIHOWSWINTEUTEDTHERARTHLEGANRYINCDASSNLFTANROGIAUNLOMOSLOIGELBLEYTTNHASNMRDEVAUESARTHEWINVARUETHNFGHATMOSCODEULEOITNHACTHOTTNIETFIRMNFLDCEOROTPESTDASGIOUEORDOTMNISTDEOTHAROPIEOKMATHTIODATANRTHERARTHIOASEDOREMSTORDOIDTHELEGANRMOSEXALEDARDASGIOUETNERDFIETHEHOIDORDDORGEINFSMNIKNYSFPDFARGTHEUOLEDNRAAARTHEPLEOKMALDEIRESSESNYUOLEDNRAOSFETNRAFSNIDEIEDTHERARTHTNCOIUHTNEPNIOUFCMHAUHTHEBMEIETNCOKETHEAIPOSEYNIIOADSARTNUOLEDNRAOTHEFRSWNKERIEOSNRYNITHEAIEXALEMOSTHOTARTELLAGERUEIEWNITSSFGGESTEDTHOTTHASMOSMHEIETHEBCAGHTYARDTHEAILNSTOQFALOTHNSEMHNMNFLDIEODNRCFSTYNLLNMCBYOATHYFLSLOVETAINMHNYOUESOGIOVETOSKWEIHOWSTHELOIGESTTNDOTEOSHETIOVELSTNUNRUEOLTHEYNFITHUHOWTEINYTHASTIOGAUTOLE'"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "scb = sanitise(cb)\n",
+    "scbp = chunks(scb, 2)\n",
+    "order = 'xlcdm'\n",
+    "ltrs = 'etoanisrhdlufcmwgybpvkxqz'\n",
+    "prs =  [p[0] for p in collections.Counter(chunks(sanitise(cb), 2)).most_common()]\n",
+    "trans = {pr[1]: pr[0].upper() for pr in zip(ltrs, prs)}\n",
+    "scbpt = cat(trans[p] for p in scbp)\n",
+    "scbpt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'MHALEATASTIFETHOTSFETNRAFSSETFWTHEYAROLPOTTLEMATHPNFDAUUOOTMOTLARGSTIEETHEHODREATHEITHETINNWSRNITHETOUTAUOLOPALATBTNNVEIUNCETHEHFGERFCEIAUOLDASODVORTOGEYOUEDPBHASOICBMATHNFTTHESFWWNITNYOGIAUNLOORDHASTOLERTEDSUNFTSDEYEOTMNFLDHOVEPEERAREVATOPLEORDMATHATTHELNSSNYPIATORRAOHOVARGIEOLASEDTHOTTHEAUERAMEIEIEODARGHASUNCCFRAUOTANRSOGIAUNLOOIIORGEDTNWLORTYOLSEARYNICOTANROPNFTTHEWLORREDDASWNSATANRNYTINNWSSETTARGOTIOWARTNMHAUHTHEAUERAYELLATASSTALLFRULEOIHNMTHESOVOGETIAPESHODLEOIREDTNDEUAWHEINFIOICBSUNCCFRAUOTANRSOSTHEBHODRNMIATARGNYTHEAINMRHNMEVEIEVEIBOICBHOSATSTIOATNISORDATSEECSLAKELBTNCETHOTNRENICNIENYTHELNUOLTIAPESWENWLETFIREDSNCENRESHEODLNVENICNREBATASOLLTIEOUHEIBPFTTHASOUTTHIEOTEREDTNFRDEICARETHEECWAIESERTAIESBSTECNYSEUFIEUNCCFRAUOTANRTHEUOESOIUAWHEIWEIHOWSCNIETHORINODSHODEROPLEDTHEECWAIETNYFRUTANRSEUFIELBORDATSLNSSMOSODASOSTEITFIRARGTHOTLNSSARTNOVAUTNIBNRTHEPOTTLEYAELDDESEIVEDCNIEIEUNGRATANRTHORATIEUEAVEDWEIHOWSSFETNRAFSREVEIKREMNIWEIHOWSHEYOALEDTNFRDEISTORDTHEUFRRARGNYTHEWLORPFTEATHEIMOBDESWATETHEVAUTNIBOGIAUNLOORDTHERARTHLEGANRIECOAREDARDASGIOUENRHEOIARGTHASREMSULOFDAFSUOESOIOFGFSTFSGEICORAUFSREINECWEINIASSFEDOSEUIETWINULOCOTANRESTOPLASHARGOUAWHEISUHNNLTNDEVELNWOREMSBSTECNYACWEIAOLUAWHEISATMNIKEDARSEUIETYNINVEITMERTBBEOISDEVELNWARGREMCETHNDSTNSEUFIEUNCCFRAUOTANRSOUINSSTHEECWAIESTFDBARGTHEMNIKSNYTHEGIEEKSOGESEFULADORDHBWOTAOORDEVERNLDEIMNIKSYINCORUAERTPOPBLNRTHESUHNNLDEVELNWEDREMMOBSTNSOYEGFOIDUNCCFRAUOTANRSYINCINCESERECAESTHESEREMSBSTECSMEIEMIATTERDNMRORDDASTIAPFTEDARTHEUNDEXNUUFLTNIFCWEIHOWSTHECNSTSTIAUTLBGFOIDEDDNUFCERTAROLLHASTNIBMHERATMOSYAROLLBUNCWLETEDCORBBEOISLOTEITHEECWEINIDNCATAORASSFEDOREXEUFTAVENIDEIMHAUHWIEYOUEDEVEIBUNWBNYTHEUNDEXTHOTATMOSTNPEGFOIDEDPBEVEIBLEGANRTNTHELOSTCORTHEDEYEOTNYTHEAUERAORDTHESFAUADENYPNFDAUUOWEIHOWSWINTEUTEDTHERARTHLEGANRYINCDASSNLFTANROGIAUNLOMOSLOIGELBLEYTTNHASNMRDEVAUESARTHEWINVARUETHNFGHATMOSCODEULEOITNHACTHOTTNIETFIRMNFLDCEOROTPESTDASGIOUEORDOTMNISTDEOTHAROPIEOKMATHTIODATANRTHERARTHIOASEDOREMSTORDOIDTHELEGANRMOSEXALEDARDASGIOUETNERDFIETHEHOIDORDDORGEINFSMNIKNYSFPDFARGTHEUOLEDNRAAARTHEPLEOKMALDEIRESSESNYUOLEDNRAOSFETNRAFSNIDEIEDTHERARTHTNCOIUHTNEPNIOUFCMHAUHTHEBMEIETNCOKETHEAIPOSEYNIIOADSARTNUOLEDNRAOTHEFRSWNKERIEOSNRYNITHEAIEXALEMOSTHOTARTELLAGERUEIEWNITSSFGGESTEDTHOTTHASMOSMHEIETHEBCAGHTYARDTHEAILNSTOQFALOTHNSEMHNMNFLDIEODNRCFSTYNLLNMCBYOATHYFLSLOVETAINMHNYOUESOGIOVETOSKWEIHOWSTHELOIGESTTNDOTEOSHETIOVELSTNUNRUEOLTHEYNFITHUHOWTEINYTHASTIOGAUTOLE'"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trhc = ''.maketrans('abmdefghijklcnopqrstuvwxyz', string.ascii_lowercase)\n",
+    "scbpt.translate(trhc)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def polybius_grid(keyword, column_order, row_order, letters_to_merge=None,\n",
+    "                  wrap_alphabet=KeywordWrapAlphabet.from_a):\n",
+    "    alphabet = keyword_cipher_alphabet_of(keyword, wrap_alphabet=wrap_alphabet)\n",
+    "    if letters_to_merge is None: \n",
+    "        letters_to_merge = {'j': 'i'}\n",
+    "    grid = {l: k \n",
+    "            for k, l in zip([(c, r) for c in column_order for r in row_order],\n",
+    "                [l for l in alphabet if l not in letters_to_merge])}\n",
+    "    for l in letters_to_merge:\n",
+    "        grid[l] = grid[letters_to_merge[l]]\n",
+    "    return grid        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'a': ('x', 'x'),\n",
+       " 'b': ('x', 'l'),\n",
+       " 'c': ('x', 'c'),\n",
+       " 'd': ('x', 'd'),\n",
+       " 'e': ('x', 'm'),\n",
+       " 'f': ('l', 'x'),\n",
+       " 'g': ('l', 'l'),\n",
+       " 'h': ('l', 'c'),\n",
+       " 'i': ('l', 'd'),\n",
+       " 'j': ('l', 'd'),\n",
+       " 'k': ('l', 'm'),\n",
+       " 'l': ('c', 'x'),\n",
+       " 'm': ('c', 'l'),\n",
+       " 'n': ('c', 'c'),\n",
+       " 'o': ('c', 'd'),\n",
+       " 'p': ('c', 'm'),\n",
+       " 'q': ('d', 'x'),\n",
+       " 'r': ('d', 'l'),\n",
+       " 's': ('d', 'c'),\n",
+       " 't': ('d', 'd'),\n",
+       " 'u': ('d', 'm'),\n",
+       " 'v': ('m', 'x'),\n",
+       " 'w': ('m', 'l'),\n",
+       " 'x': ('m', 'c'),\n",
+       " 'y': ('m', 'd'),\n",
+       " 'z': ('m', 'm')}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_grid('', order, order)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'a': ('a', 'a'),\n",
+       " 'b': ('a', 'b'),\n",
+       " 'c': ('a', 'c'),\n",
+       " 'd': ('a', 'd'),\n",
+       " 'e': ('a', 'e'),\n",
+       " 'f': ('b', 'a'),\n",
+       " 'g': ('b', 'b'),\n",
+       " 'h': ('b', 'c'),\n",
+       " 'i': ('b', 'd'),\n",
+       " 'j': ('b', 'd'),\n",
+       " 'k': ('b', 'e'),\n",
+       " 'l': ('c', 'a'),\n",
+       " 'm': ('c', 'b'),\n",
+       " 'n': ('c', 'c'),\n",
+       " 'o': ('c', 'd'),\n",
+       " 'p': ('c', 'e'),\n",
+       " 'q': ('d', 'a'),\n",
+       " 'r': ('d', 'b'),\n",
+       " 's': ('d', 'c'),\n",
+       " 't': ('d', 'd'),\n",
+       " 'u': ('d', 'e'),\n",
+       " 'v': ('e', 'a'),\n",
+       " 'w': ('e', 'b'),\n",
+       " 'x': ('e', 'c'),\n",
+       " 'y': ('e', 'd'),\n",
+       " 'z': ('e', 'e')}"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_grid('a', 'abcde', 'abcde')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_grid('elephant', 'abcde', 'abcde')['b'] == ('b', 'c')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_grid('elephant', 'abcde', 'abcde')['e'] == ('a', 'a')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def polybius_reverse_grid(keyword, column_order, row_order, letters_to_merge=None,\n",
+    "                  wrap_alphabet=KeywordWrapAlphabet.from_a):\n",
+    "    alphabet = keyword_cipher_alphabet_of(keyword, wrap_alphabet=wrap_alphabet)\n",
+    "    if letters_to_merge is None: \n",
+    "        letters_to_merge = {'j': 'i'}\n",
+    "    grid = {k: l \n",
+    "            for k, l in zip([(c, r) for c in column_order for r in row_order],\n",
+    "                [l for l in alphabet if l not in letters_to_merge])}\n",
+    "#     for l in letters_to_merge:\n",
+    "#         for r, c in grid:\n",
+    "#             if grid[r, c] == letters_to_merge[l]:\n",
+    "#                 grid[l] = grid[r, c]\n",
+    "    return grid        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{('c', 'c'): 'n',\n",
+       " ('c', 'd'): 'o',\n",
+       " ('c', 'l'): 'm',\n",
+       " ('c', 'm'): 'p',\n",
+       " ('c', 'x'): 'l',\n",
+       " ('d', 'c'): 's',\n",
+       " ('d', 'd'): 't',\n",
+       " ('d', 'l'): 'r',\n",
+       " ('d', 'm'): 'u',\n",
+       " ('d', 'x'): 'q',\n",
+       " ('l', 'c'): 'h',\n",
+       " ('l', 'd'): 'i',\n",
+       " ('l', 'l'): 'g',\n",
+       " ('l', 'm'): 'k',\n",
+       " ('l', 'x'): 'f',\n",
+       " ('m', 'c'): 'x',\n",
+       " ('m', 'd'): 'y',\n",
+       " ('m', 'l'): 'w',\n",
+       " ('m', 'm'): 'z',\n",
+       " ('m', 'x'): 'v',\n",
+       " ('x', 'c'): 'c',\n",
+       " ('x', 'd'): 'd',\n",
+       " ('x', 'l'): 'b',\n",
+       " ('x', 'm'): 'e',\n",
+       " ('x', 'x'): 'a'}"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_reverse_grid('', order, order)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def polybius_flatten(pair, column_first):\n",
+    "    if column_first:\n",
+    "        return str(pair[1]) + str(pair[0])\n",
+    "    else:\n",
+    "        return str(pair[0]) + str(pair[1])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def polybius_encipher(message, keyword, column_order, row_order, \n",
+    "                      column_first=False,\n",
+    "                      letters_to_merge=None, wrap_alphabet=KeywordWrapAlphabet.from_a):    \n",
+    "    grid = polybius_grid(keyword, column_order, row_order, letters_to_merge, wrap_alphabet)\n",
+    "    return cat(polybius_flatten(grid[l], column_first)\n",
+    "               for l in message\n",
+    "               if l in grid)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'mllcldcxxmldddlddcdddldmxm'"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_encipher('while it is true', '', order, order)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'mllcldcxxmldddlddcdddldmxmddlcxxdddcdmxmddcdcclddmdcdcxmdddmcmddlcxmlxldccxxcxxlxxdd'"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "sanitise(cb[:100])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def polybius_decipher(message, keyword, column_order, row_order, \n",
+    "                      column_first=False,\n",
+    "                      letters_to_merge=None, wrap_alphabet=KeywordWrapAlphabet.from_a):    \n",
+    "    grid = polybius_reverse_grid(keyword, column_order, row_order, letters_to_merge, wrap_alphabet)\n",
+    "    column_index_type = type(column_order[0])\n",
+    "    row_index_type = type(row_order[0])\n",
+    "    if column_first:\n",
+    "        pairs = [(column_index_type(p[1]), row_index_type(p[0])) for p in chunks(message, 2)]\n",
+    "    else:\n",
+    "        pairs = [(row_index_type(p[0]), column_index_type(p[1])) for p in chunks(message, 2)]\n",
+    "    return cat(grid[p] for p in pairs if p in grid)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'whileitistrue'"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_decipher('mllcldcxxmldddlddcdddldmxm', '', order, order)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'lmcldlxcmxdldddlcdddldmdmx'"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_encipher('whileitistrue', '', order, order, column_first=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'kmrcvrtrotiyv'"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_decipher('lmcldlxcmxdldddlcdddldmdmx', '', order, order)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'whileitistrue'"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_decipher('lmcldlxcmxdldddlcdddldmdmx', '', order, order, column_first=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'52232431152444244344424515'"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_encipher('while it is true', '', '12345', '12345')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'52232431152444244344424515'"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_encipher('while it is true', '', [1, 2, 3, 4, 5], [1, 2, 3, 4, 5])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'whileitistrue'"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_decipher('52232431152444244344424515', '', [1, 2, 3, 4, 5], [1, 2, 3, 4, 5])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from multiprocessing import Pool\n",
+    "\n",
+    "def polybius_break_mp(message, column_labels, row_labels,\n",
+    "                      letters_to_merge=None,\n",
+    "                      wordlist=keywords, fitness=Pletters,\n",
+    "                      number_of_solutions=1, chunksize=500):\n",
+    "    \"\"\"Breaks a Polybius substitution cipher using a dictionary and\n",
+    "    frequency analysis\n",
+    "\n",
+    "    >>> polybius_break_mp(polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'abcde'), \\\n",
+    "          'abcde', 'abcde', \\\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS\n",
+    "    (('elephant', <KeywordWrapAlphabet.from_a: 1>, 'abcde', 'abcde', False), \\\n",
+    "    -54.5397...)\n",
+    "    >>> polybius_break_mp(polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'abcde', column_first=True), \\\n",
+    "          'abcde', 'abcde', \\\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS\n",
+    "    (('elephant', <KeywordWrapAlphabet.from_a: 1>, 'abcde', 'abcde', True), \\\n",
+    "    -54.5397...)\n",
+    "    >>> polybius_break_mp(polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'abcde', column_first=False), \\\n",
+    "          'abcde', 'abcde', \\\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS\n",
+    "    (('elephant', <KeywordWrapAlphabet.from_a: 1>, 'abcde', 'abcde', False), \\\n",
+    "    -54.5397...)\n",
+    "    >>> polybius_break_mp(polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'pqrst', column_first=True), \\\n",
+    "          'abcde', 'pqrst', \\\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS\n",
+    "    (('elephant', <KeywordWrapAlphabet.from_a: 1>, 'abcde', 'pqrst', True), \\\n",
+    "    -54.5397...)\n",
+    "    \"\"\"\n",
+    "    if letters_to_merge is None: \n",
+    "        letters_to_m53880erge = {'j': 'i'}\n",
+    "    with Pool() as pool:\n",
+    "        helper_args = [(message, word, wrap, \n",
+    "                        column_labels, row_labels, column_first, \n",
+    "                        letters_to_merge, \n",
+    "                        fitness)\n",
+    "                       for word in wordlist\n",
+    "                       for wrap in KeywordWrapAlphabet\n",
+    "                       for column_first in [False, True]]\n",
+    "        # Gotcha: the helper function here needs to be defined at the top level\n",
+    "        #   (limitation of Pool.starmap)\n",
+    "        breaks = pool.starmap(polybius_break_worker, helper_args, chunksize)\n",
+    "        if number_of_solutions == 1:\n",
+    "            return max(breaks, key=lambda k: k[1])\n",
+    "        else:\n",
+    "            return sorted(breaks, key=lambda k: k[1], reverse=True)[:number_of_solutions]\n",
+    "\n",
+    "def polybius_break_worker(message, keyword, wrap_alphabet, \n",
+    "                          column_order, row_order, column_first, \n",
+    "                          letters_to_merge, \n",
+    "                          fitness):\n",
+    "    plaintext = polybius_decipher(message, keyword, \n",
+    "                                  column_order, row_order, \n",
+    "                                  column_first=column_first,\n",
+    "                                  letters_to_merge=letters_to_merge, \n",
+    "                                  wrap_alphabet=wrap_alphabet)\n",
+    "    if plaintext:\n",
+    "        fit = fitness(plaintext)\n",
+    "    else:\n",
+    "        fit = float('-inf')\n",
+    "    logger.debug('Polybius break attempt using key {0} (wrap={1}, merging {2}), '\n",
+    "                 'columns as {3}, rows as {4} (column_first={5}) '\n",
+    "                 'gives fit of {6} and decrypt starting: '\n",
+    "                 '{7}'.format(keyword, wrap_alphabet, letters_to_merge,\n",
+    "                              column_order, row_order, column_first,\n",
+    "                              fit, sanitise(plaintext)[:50]))\n",
+    "    return (keyword, wrap_alphabet, column_order, row_order, column_first), fit"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 2.67 s, sys: 308 ms, total: 2.98 s\n",
+      "Wall time: 3min 58s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(('a', <KeywordWrapAlphabet.from_a: 1>, 'xlcdm', 'xlcdm', False),\n",
+       " -3018.8648333417113)"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%time polybius_break_mp(sanitise(cb), order, order)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(('elephant',\n",
+       "  <KeywordWrapAlphabet.from_a: 1>,\n",
+       "  [1, 2, 3, 4, 5],\n",
+       "  [1, 2, 3, 4, 5],\n",
+       "  True),\n",
+       " -54.53880323982303)"
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_break_mp(polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', \\\n",
+    "          [1, 2, 3, 4, 5], [1, 2, 3, 4, 5], \\\n",
+    "          KeywordWrapAlphabet.from_last), \\\n",
+    "          [1, 2, 3, 4, 5], [1, 2, 3, 4, 5], \\\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'2214445544551522115522511155551543114252542214111352123234442355411135441314115451112122'"
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', \\\n",
+    "          [1, 2, 3, 4, 5], [1, 2, 3, 4, 5], \\\n",
+    "          wrap_alphabet=KeywordWrapAlphabet.from_last)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(('elephant',\n",
+       "  <KeywordWrapAlphabet.from_last: 2>,\n",
+       "  [1, 2, 3, 4, 5],\n",
+       "  [1, 2, 3, 4, 5],\n",
+       "  False),\n",
+       " -54.53880323982303)"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_break_mp('2214445544551522115522511155551543114252542214111352123234442355411135441314115451112122', \\\n",
+    "          [1, 2, 3, 4, 5], [1, 2, 3, 4, 5], \\\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'thisisatestmessageforthepolybiusdecipherment'"
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_decipher('2214445544551522115522511155551543114252542214111352123234442355411135441314115451112122', \\\n",
+    "                  'elephant',\n",
+    "          [1, 2, 3, 4, 5], [1, 2, 3, 4, 5], \\\n",
+    "          wrap_alphabet=KeywordWrapAlphabet.from_last)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{(1, 1): 'e',\n",
+       " (1, 2): 'l',\n",
+       " (1, 3): 'p',\n",
+       " (1, 4): 'h',\n",
+       " (1, 5): 'a',\n",
+       " (2, 1): 'n',\n",
+       " (2, 2): 't',\n",
+       " (2, 3): 'b',\n",
+       " (2, 4): 'c',\n",
+       " (2, 5): 'd',\n",
+       " (3, 1): 'f',\n",
+       " (3, 2): 'g',\n",
+       " (3, 3): 'i',\n",
+       " (3, 4): 'k',\n",
+       " (3, 5): 'm',\n",
+       " (4, 1): 'o',\n",
+       " (4, 2): 'q',\n",
+       " (4, 3): 'r',\n",
+       " (4, 4): 's',\n",
+       " (4, 5): 'u',\n",
+       " (5, 1): 'v',\n",
+       " (5, 2): 'w',\n",
+       " (5, 3): 'x',\n",
+       " (5, 4): 'y',\n",
+       " (5, 5): 'z'}"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_reverse_grid('elephant', [1, 2, 3, 4, 5], [1, 2, 3, 4, 5] )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "type(2)('4')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'bbadccddccddaebbaaddbbceaaddddaecbaacadadcbbadaaacdaabedbcccdeddbeaabdccacadaadcceaababb'"
+      ]
+     },
+     "execution_count": 35,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'abcde', column_first=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'bbdaccddccddeabbaaddbbecaaddddeabcaaacadcdbbdaaacaadbadecbccedddebaadbcccadaaacdecaaabbb'"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'abcde', column_first=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'toisisvtestxessvbephktoefhnugiysweqifoekxelt'"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_decipher('bbdaccddccddeabbaaddbbecaaddddeabcaaacadcdbbdaaacaadbadecbccedddebaadbcccadaaacdecaaabbb', 'elephant', 'abcde', 'abcde', column_first=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'thisisatestmessageforthepolybiusdecipherment'"
+      ]
+     },
+     "execution_count": 38,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_decipher('bbdaccddccddeabbaaddbbecaaddddeabcaaacadcdbbdaaacaadbadecbccedddebaadbcccadaaacdecaaabbb', 'elephant', 'abcde', 'abcde', column_first=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(('elephant', <KeywordWrapAlphabet.from_a: 1>, 'abcde', 'abcde', False),\n",
+       " -54.53880323982303)"
+      ]
+     },
+     "execution_count": 39,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_break_mp(polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'abcde'), \\\n",
+    "          'abcde', 'abcde',\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(('elephant', <KeywordWrapAlphabet.from_a: 1>, 'abcde', 'abcde', True),\n",
+       " -54.53880323982303)"
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_break_mp(polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'abcde', column_first=True), \\\n",
+    "          'abcde', 'abcde',\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(('elephant', <KeywordWrapAlphabet.from_a: 1>, 'abcde', 'abcde', False),\n",
+       " -54.53880323982303)"
+      ]
+     },
+     "execution_count": 41,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_break_mp(polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'abcde', column_first=False), \\\n",
+    "          'abcde', 'abcde',\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(('elephant', <KeywordWrapAlphabet.from_a: 1>, 'abcde', 'pqrst', True),\n",
+       " -54.53880323982303)"
+      ]
+     },
+     "execution_count": 42,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_break_mp(polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'pqrst', column_first=True), \\\n",
+    "          'abcde', 'pqrst',\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'bwaycxdycxdyazbwavdybwczavdydyazcwavcvdvdxbwayavaxdvaweybxcxdzdybzavbycxaxayavdxczavbvbw'"
+      ]
+     },
+     "execution_count": 43,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_encipher('this is a test message for the ' \\\n",
+    "          'polybius decipherment', 'elephant', 'abcde', 'vwxyz')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'thisisatestmessageforthepolybiusdecipherment'"
+      ]
+     },
+     "execution_count": 44,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_decipher('bwaycxdycxdyazbwavdybwczavdydyazcwavcvdvdxbwayavaxdvaweybxcxdzdybzavbycxaxayavdxczavbvbw', \n",
+    "                  'elephant', 'abcde', 'vwxyz')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "logger.setLevel(logging.DEBUG)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(('elephant', <KeywordWrapAlphabet.from_a: 1>, 'abcde', 'vwxyz', False),\n",
+       " -54.53880323982303)"
+      ]
+     },
+     "execution_count": 46,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "polybius_break_mp('bwaycxdycxdyazbwavdybwczavdydyazcwavcvdvdxbwayavaxdvaweybxcxdzdybzavbycxaxayavdxczavbvbw', \\\n",
+    "          'abcde', 'vwxyz',\n",
+    "          wordlist=['cat', 'elephant', 'kangaroo'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 47,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "logger.debug('test')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}