Started 2015 challenges

diff --git a/.directory b/.directory
new file mode 100644 (file)
index 0000000..280ac2b
--- /dev/null
+++ b/.directory
@@ -0,0 +1,4 @@
+[Dolphin]
+Timestamp=2015,10,16,9,24,56
+Version=3
+ViewMode=1

diff --git a/2015-challenge1.ipynb b/2015-challenge1.ipynb
new file mode 100644 (file)
index 0000000..f565f7b
--- /dev/null
+++ b/2015-challenge1.ipynb
@@ -0,0 +1,138 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "%matplotlib inline\n",
+    "\n",
+    "from cipherbreak import *\n",
+    "\n",
+    "c1a = open('2015/1a.ciphertext').read()\n",
+    "c1b = open('2015/1b.ciphertext').read()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(12, -883.4816832492597)"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "key_a, score = caesar_break(c1a)\n",
+    "key_a, score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "HARRY, SORRY TO DRAG YOU BACK IN , WE WERE HOPING TO GIVE YOU SOME TIME OFF AFTER THE LAST CASE, BUT SOMETHING CAME UP AND WE NEED YOUR HELP. \n",
+      "\n",
+      "AT A MEETING OF THE FOUR POWERS ALLIED CONTROL COUNCIL TWO WEEKS AGO THE FRENCH ACCUSED THE RUSSIANS OF SHELTERING A NAZI MEDIC KNOWN AS THE REICHSDOKTOR. APPARENTLY THEY INTERCEPTED A MORSE CODE RADIO BROADCAST FROM THE RUSSIAN SECTOR OF BERLIN IN WHICH THE DOCTOR WAS OFFERING INTELLIGENCE ABOUT THE RATLINES IN EXCHANGE FOR ASYLUM. THE RUSSIANS CLAIMED NOT TO KNOW ANYTHING ABOUT IT, AND MAYBE THEY ARE TELLING THE TRUTH, BUT THINGS HAVE BEEN A LITTLE FROSTY SINCE TRUMAN'S SPEECH ON MARCH TWELFTH AND WE REALLY DON'T NEED MORE CONFLICT RIGHT NOW. WE FIGURE WITH YOUR CONTACTS OVER HERE YOU MIGHT BE ABLE TO FIND OUT IF THE RUSSIANS ARE TELLING THE TRUTH. I HAVE ATTACHED THE ENCRYPTED TRANSCRIPT OF THE BROADCAST. \n",
+      "CHARLIE\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(caesar_decipher(c1a, key_a))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(14, -403.53308240183975)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "key_b, score = caesar_break(c1b)\n",
+    "key_b, score"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "if you want to know the secret of the rat lines i maybe able to help but the price will be high and is not negotiable life herein berlin has lost its lustre and i want sanctuary in a more congenial climate with security for my future i can provide details of personnel policy security and routes and can furnish you with documentary evidence of the reach of the organization the reichs doktor\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(' '.join(segment(sanitise(caesar_decipher(c1b, key_b)))))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.4.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}

diff --git a/2015/1a.ciphertext b/2015/1a.ciphertext
new file mode 100644 (file)
index 0000000..d222977
--- /dev/null
+++ b/2015/1a.ciphertext
@@ -0,0 +1,4 @@
+TMDDK, EADDK FA PDMS KAG NMOW UZ , IQ IQDQ TABUZS FA SUHQ KAG EAYQ FUYQ ARR MRFQD FTQ XMEF OMEQ, NGF EAYQFTUZS OMYQ GB MZP IQ ZQQP KAGD TQXB. 
+
+MF M YQQFUZS AR FTQ RAGD BAIQDE MXXUQP OAZFDAX OAGZOUX FIA IQQWE MSA FTQ RDQZOT MOOGEQP FTQ DGEEUMZE AR ETQXFQDUZS M ZMLU YQPUO WZAIZ ME FTQ DQUOTEPAWFAD. MBBMDQZFXK FTQK UZFQDOQBFQP M YADEQ OAPQ DMPUA NDAMPOMEF RDAY FTQ DGEEUMZ EQOFAD AR NQDXUZ UZ ITUOT FTQ PAOFAD IME ARRQDUZS UZFQXXUSQZOQ MNAGF FTQ DMFXUZQE UZ QJOTMZSQ RAD MEKXGY. FTQ DGEEUMZE OXMUYQP ZAF FA WZAI MZKFTUZS MNAGF UF, MZP YMKNQ FTQK MDQ FQXXUZS FTQ FDGFT, NGF FTUZSE TMHQ NQQZ M XUFFXQ RDAEFK EUZOQ FDGYMZ’E EBQQOT AZ YMDOT FIQXRFT MZP IQ DQMXXK PAZ’F ZQQP YADQ OAZRXUOF DUSTF ZAI. IQ RUSGDQ IUFT KAGD OAZFMOFE AHQD TQDQ KAG YUSTF NQ MNXQ FA RUZP AGF UR FTQ DGEEUMZE MDQ FQXXUZS FTQ FDGFT. U TMHQ MFFMOTQP FTQ QZODKBFQP FDMZEODUBF AR FTQ NDAMPOMEF. 
+OTMDXUQ
\ No newline at end of file

diff --git a/2015/1b.ciphertext b/2015/1b.ciphertext
new file mode 100644 (file)
index 0000000..98a9dd3
--- /dev/null
+++ b/2015/1b.ciphertext
@@ -0,0 +1 @@
+WT MCI KOBH HC YBCK HVS GSQFSH CT HVS FOHZWBSG W AOM PS OPZS HC VSZD PIH HVS DFWQS KWZZ PS VWUV OBR WG BCH BSUCHWOPZS ZWTS VSFS WB PSFZWB VOG ZCGH WHG ZIGHFS OBR W KOBH GOBQHIOFM WB O ACFS QCBUSBWOZ QZWAOHS KWHV GSQIFWHM TCF AM TIHIFS W QOB DFCJWRS RSHOWZG CT DSFGCBBSZ DCZWQM GSQIFWHM OBR FCIHSG OBR QOB TIFBWGV MCI KWHV RCQIASBHOFM SJWRSBQS CT HVS FSOQV CT HVS CFUOBWNOHWCB HVS FSWQVGRCYHCF 

diff --git a/SIGNED.md b/SIGNED.md
index 3a017344873df589901f123756e6003ae4db4307..68e6966baeee591db89dd3cbb4d6623b4456518d 100644 (file)
--- a/SIGNED.md
+++ b/SIGNED.md
@@ -3,19 +3,19 @@
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v2
 
-iQIcBAABAgAGBQJU6u7XAAoJEJPB2e07Pgbqj+4P/RQk28yTg894dSybAhs3kNtS
-knFx9OYSkv+PkUHAMq6HglIVG4T1n4237tekNSVzMFaummaQ+VRVFprqYgcvh7ol
-4z+i6BWazou6sSa8qHJviNeF4oj+cnqzVlSWfPhdRHcjxathxyaFockaZagGeA1P
-egKPF3WXpq+rR6nFDXMLq7kdAyyLIwNVcepr9ieOLeKvx5rdi8dTFssvsXIJ37DZ
-Tl6ynEGpvkhZZrfm/vsFp79uDqT+XfP1cKM7Rm7jmYa5hiPgLE3kM5wOEpmUMXvD
-PzSjJ5yuUm82p6qHQW4BK+IFTdPFbL0ErziVP7vvWlFX2iilZ9sE2EEMvh/tCwmT
-K9vDJk2l0CkhFOOkih7O7tmxF2a27LrxpGtqv7Kw/w7+1LhwhoEwVLnXNl/PS4JM
-iT7QvkK3mYCcB9IqUPNmCs3doqSuOqoRMUFdYxj0qOlQgqNukc5U5ETX52goGpB6
-huxD5FyFgznn5YAaYIqklO81z1YL2R0CZ54CoIx027hKHvo98JEN9g0kxBpoo7AK
-JcphW4z+EOQtNtHHsXoifp/wRiChBzhE1QMXw5OSeMpvO7p8zPYS6poO2kn+kzsz
-6f8X7/4PrmusahqFZ5w3xEbmEdxsvpmhhXYgAdh1dWtBAMY+DIwn5IaVccvftsHx
-/ERD+mLkHcWviY54Kawy
-=fVdI
+iQIcBAABCAAGBQJWILouAAoJEJPB2e07PgbqryEQAI0JcUaIHReXj+R9fbd2XsgN
+VezfumlVz/PXtbHP39grBqWwxWEwqhh6qZBdBpImVYR5xUmYtWL/5zsNFY0iQwMS
+TmBgVn6YH9j9iQ7MpZUscO71P2pwhKqwGEcb7NxcHr4HocmmTrHdhVmPL27suLwC
+NDYyy1SY4vCoxBaMbY4rbTcUHs/jY0tuIyEX7q6JByM07Rune+p3OxtYyjx3HBmz
+pQy7Cs1yJf5vb4BZuhjB3izyI8H6MIhc1oupBn1UTxKW5I4yCk6bFBT+t/dJGBYV
+D6FD3fSgjwKCqtgkJ7sxFpHiqUud9UCxDRW6YyQqSZd+nW7SJjXX7aepMj6giMRW
++O1sLJoid3aQkKKHcWDp5/Nb3sl+xBQx8MfVLZUFGFU78UUwf09ZNIfEuNk5ZXdC
+1MVyNHothCFas7AoDjLMnOAtUucs71ZwT0AMMQFxPGDT7a8z2Qexxs7Wy1j8DY6E
+joLvLIyJsZF+NKKEWi1FsPy2DuGfqG+0m6qWo2rMz/ugV6f8pyVfIDVdkDNpE9uN
+Rn3t7D6u+XMGlmnmx6lAHVsDLBlOB5zS7LyBsiGjkhcFc+PSL+Zv4MlTBIZQyLeD
+HyzhUtVFGsTBy1sZOaazipdC1+74X0I5OiUVoSCPcNOsFGTqfU0+tjXCFTwgpQEb
+EfQreGq0WtV1zoNFVsLN
+=mndn
 -----END PGP SIGNATURE-----
 
 ```
@@ -29,6 +29,7 @@ JcphW4z+EOQtNtHHsXoifp/wRiChBzhE1QMXw5OSeMpvO7p8zPYS6poO2kn+kzsz
 ```
 size     exec  file                                               contents                                                                                                                         
                ./                                                                                                                                                                                  
+60               .directory                                       0e83b2cb4dc042236e38c6f8b2575dd0e30b70d4d396795008d5fb3facea2872                                                                 
 384              .gitignore                                       a93de2ae5c2a47a38599751d1f914566569dfa09dd1778e207117db6c71421dd                                                                 
                  2012/                                                                                                                                                                             
 1678               1a.ciphertext                                  c7c16198650e7d91577683acca664a0e588ac5474046a028dea3d9dc4b388df8                                                                 
@@ -111,6 +112,10 @@ size     exec  file                                               contents
 40307            2014-challenge6.ipynb                            5085a4ae2a562c87983cbff817ac89826973341551980563ae0c77ab747a5dff                                                                 
 44472            2014-challenge7.ipynb                            9800d39c881f22f67f1182f6ca73bbf40a123d0deb598a6009f545f421955241                                                                 
 188080           2014-challenge8.ipynb                            c59bcfbb39e48e6e151cc0c86fe48add67844954cf3c65d560bbda0c972a8376                                                                 
+                 2015/                                                                                                                                                                             
+874                1a.ciphertext                                  5d0d71f24522e1c05127feb601bf8dc82566f3d2795fb51c4a60e22d900bb5eb                                                                 
+394                1b.ciphertext                                  a35e8265aea45ab5f7c5eb141e5e65c85ecc24c7d87f3e8197dfe6795a7d3226                                                                 
+3620             2015-challenge1.ipynb                            2ab544f48c22a3a2e665b03ae094ac2de04aee8bdb37366209276a937d248d6a                                                                 
 18025            LICENSE                                          a01259a1b522cf0de95824f9860613b453153eebac468e96196d5d7dba84786c                                                                 
 7999             LJ!-Qt!-Fghxft-dferts%3B-hsjeukaxxn-sfedw.ipynb  429b6c6995096ff19c28a5ee342bef8ea4774200bdf9aaf6268de3cb8b28df28                                                                 
 61               README.md                                        277247b410300ee16477b12ca54ad878d81c8061f6134e2e1cadccaf299de3a3                                                                 
@@ -131,7 +136,7 @@ size     exec  file                                               contents
 3027             find_best_caesar_break_parameters.py             0347d80309179d937a88fd1c8684490a513ccd086366c5a0dd55b8a2fe5c565f                                                                 
 1236             find_wikipedia_titles.py                         f040bf855dfec7fff9d8e5eba2fb509179bc53bc02a20b26b7fc61fef983aa45                                                                 
 37128            hill-ciphers.ipynb                               ce802c2be807b4565858b568d3a82c65a3957aa625344189f8f2a055237b3fdd                                                                 
-5401             language_models.py                               e4f868b0322ac57fc4027c22c3c6a4d6b24a30894d19110f1a28f11bec0cbd04                                                                 
+5516             language_models.py                               9f6c60892b385a8443202edda95074eb28e2b66981e25e27f4153b887e2c9ee3                                                                 
 368              lettercount.py                                   ed36497d62cf75b91994055e4a18848b2fabe5ce793cd76a77fabfc94d81d4f3                                                                 
 592              make-cracking-dictionary.py                      71791e64e4853cd9ca292cb436bbe8c72dd60f509811174df93ed2067683d5c1                                                                 
 7077             norms.py                                         a657a36c1741e6f3a513386b318fcc99e6b11f98ec64a48284b47462ff2acf30                                                                 

diff --git a/language_models.py b/language_models.py
index 929746888d036fb54de3f1fbf228e296e0bcd027..19f886fcefcb4384184e0bbad108e6925f029bbf 100644 (file)
--- a/language_models.py
+++ b/language_models.py
@@ -6,6 +6,8 @@ import unicodedata
 import itertools
 from math import log10
 
+unaccent_specials = ''.maketrans({"’": "'"})
+
 def letters(text):
     """Remove all non-alphabetic characters from a text
     >>> letters('The Quick')
@@ -31,7 +33,8 @@ def unaccent(text):
     >>> unaccent('HÉLLÖ')
     'HELLO'
     """
-    return unicodedata.normalize('NFKD', text).\
+    translated_text = text.translate(unaccent_specials)
+    return unicodedata.normalize('NFKD', translated_text).\
         encode('ascii', 'ignore').\
         decode('utf-8')