/tmp
/__pycache__/*
*pyc
+
-HELMUTS COUSINS ARE I SUPPOSE KIND IN THEIR OWN WAY BUT THERE IS LITTLE WARMTH IN THE KINDNESS I RECEIVE ANNA TRIES TO MAKE ME COMFORTABLE BUT SHE IS AFRAID THE SS OFFICER WHO BRINGS US THE PAINTINGS IS CRUEL AND COWARDLY AND HE BEATS ANNA IF MY WORK IS NOT GOOD ENOUGH HE IS SCARED THAT IF HE BEATS ME HE MIGHT DAMAGE MY HANDS AND TOO SCARED TO BEAT HER HUSBAND DANIEL A BEAR OF A MAN WHO TOWERS OVER HIM IT DOESNT MATTER THE REAL POWER LIES WITH THE BULLY HE COULD HAVE US ALL SHOT AND WE ALL KNOW IT DANIEL SCARES ME TOO BUT ONLY BECAUSE HE REMINDS ME OF HELMUT AND THAT REMINDS ME OF THE CAMP HE NEVER SPEAKS NEVER LOOKS ME IN THE EYE AND NEVER WANTS ANYTHING FROM ME I THINK HE HATES ME FOR BRINGING THE SS TO HIS HOUSE BUT FOR ANNAS SAKE HE BRINGS ME WHAT I NEED WHAT I MOST NEED IS A WAY OUT OF HERE WHEN I AM GONE ANNAS BEATINGS WILL STOP AND MAYBE DANIEL WILL STOP HATING ME BUT I AM WATCHED ALL DAY AND THE HOUSE IS LOCKED AT NIGHT THAT WILL NOT STOP ME FROM TRYING
\ No newline at end of file
+helmut's cousins are i suppose kind in their own way but there is little warmth in the kindness i receive anna tries to make me comfortable but she is afraid the ss officer who brings us the paintings is cruel and cowardly and he beats anna if my work is not goodenough he is scared that if he beats me he might damage my hands and too scared to beat her husband daniela bear of a man who towers over him it doesnt matter the real power lies with the bully he could have us all shot and we all know it daniel scares me too but only because he reminds me of helmut and that reminds me of the camp he never speaks never looks me in the eye and never wants anything from me i think he hates me for bringing the ss to his house but for annas sake he brings me what i need what i most need is away out of here when i am gone annas beatings will stop and maybe daniel will stop hating me but i am watched all day and the house is locked at night that will not stop me from trying
\ No newline at end of file
--- /dev/null
+EVWZE AGJUU WMWHM CWCIA GAHIW CUNDC CANIW DCVAH IDZAI VAEMW CIWCU MCTID DYWII DWIMZ ROALD GAIVA PMGMC TCDDC AYCDP HPVMI VMEEA CATID WITJG WCUIV ARAMG HWIPM HBWHH WCUUW KACVD PZDCU WIPMH MPMRW NMCJC TAGHI MCTIV ALGAC NVGAZ JNIMC NAIDZ AIUDD LWIMU MWCWL DZZDP ATJER DJGZA MTDCN VMBOD GTMCT TWHND KAGAT IVMII VAZDJ KGAHE MWCIW CUHPA GAIGM CHLAG GATID ZDJKW UCRIV AMOOM RATAZ DNTWA JIVAB JHAAT ABDCI MJOMC MCTLW CMZZR IDBDC IMZJC TAGIV AHJEA GKWHW DCDLI VABJH AJBHT WGANI DGXMN FJAHX MJXMG TOJIW VMKAM LAAZW CUPAM GABWH HWCUH DBAIV WCUWC MZZIV WHXJH INMCI EJIBR LWCUA GDCWI NVMHW CUJEI VAOMN YUGDJ CTHID GRDCD JGRDJ CUEMW CIAGW IHAAB HZWYA VAGLM BWZRV MKAVM TMZDC UWCIA GAHIW CMGIG AHIDG MIWDC MCTWU JAHHI VMIWH PVAGA HVAZA MGCAT VAGHY WZZHO JIVAG DOKWD JHJCT AGHIM CTWCU DLMLD GUAGH PDGYB MYABA PDCTA GWLHD BADCA WCIVA LMBWZ RBMRO AIVAB JNVZD KATUG MCTLM IVAGV MTMZA HHDOK WDJHH DJGNA DLWCN DBANM CRDJL WCTDJ IBDGA MODJI IVABP VMIEM GIDLW IMZRT WTIVA LMBWZ RNDBA LGDBD GWUWC MZZRW PWZZC AATIV MIQGM RDLIV AEMWC IWCUM HHDDC MHRDJ NMCUA IWITD CAWVM KAMVJ CNVIV ADZTZ MTRWH VWTWC UBDGA IVMCV AGLAA ZWCUH OAVWC TIVMI HBWZA MZZIV AOAHI VMGGR
--- /dev/null
+WPSHC TSGZR LSKON JZSHJ CWONJ NTLSB TJDLN TLYDC BTSCV WXKHJ NSVJW BTJDJ KGCJN TLSCM SHSOS WCHJJ NTPSZ ZWCJN THNSV DPHWS BCDJH KGTPN SJNTH SPAKJ WYTEJ BRNSC VHATN WCVBR ASLYJ DNWVT JNTGT VEWOB TCJWN SVATT CPDGY WCOSC VNTHS WVCDJ NWCON THTTB HJDNS MTHDI JTCTV AKJWY CDPWL SCCDJ JGKHJ SCRDC TNTGT HNTBK HJCDJ ATVWH LDMTG TVNTG TDGJN TRPWZ ZJSYT SPSRB RZSHJ NDETD IGTHL KWCON TGSCV GTJKG CWCON TGJDJ NTONT JJDBR EZSCH JDTHL SETSG TDIZT HHWBE DGJSC LTAKJ BRJGS MTZVD LKBTC JHSGT CDPLD BEZTJ TJNTE SETGH SCVWC YHPTG TNSGV JDSLF KWGTA KJWTQ LKHTV BKLND IWJAR TQEZS WCWCO JNSJW CTTVT VJDHY TJLNJ DHNSG ETCBR HYWZZ HPNWL NNSVV TJTGW DGSJT VWCJN TLSBE WSBHS MWCOH KLNID DVSHB WONJZ SHJBD HJZRN SGVAG TSVSC VNSGV LNTTH TSOSW CHJJN TVSRH PNTCW NDETJ DGKCI GDBJN WHEZS LTCDP WPWZZ CTTVJ DIWCV SPSRJ DHJTS ZBDCT RJDES RIDGB RXDKG CTRBR EZSLT NWONW CJNTS JJWLO WMTHB TSMWT PDIJN TLWJR PNWLN NSHSZ ZDPTV BTJDB SYTSB SEJDO KWVTB TDCJN TBDDC ZTHHC WONJP NTCWP WZZIW CSZZR GKCSC VWYTT EJNTB SEPWJ NNTGS CVPWJ NJNWH VWSGR KCVTG JNTAD SGVHW PWZZZ WMTSC VWPWZ ZATIG TTSCV HDPWZ ZHNT
--- /dev/null
+BSTWI STHTH ISEWA HIZDH AGASH RTAGP EYIGT EHAYR TITHA IZJOS ZYEIZ REFGP BITSA KEYIS ARIZR EFGPB IAKTO EYEGE DZGAM STWEA YRTAW XZHIR TRYIO EIIST HZYEA IWEAH ITIIE WWHJH ISAIH SEOZI AMAPL JILEP ZYRIS AITAX YZIHJ GEMSA ISABB EYERM SEGER TRHSE OZRTR ISEBA TYITY OOZMT ISSEG AYRTD TIRTR SZMFZ XETIT HYZML AFVTY ISEWZ JKGER TRRAY TEWZG ISEHH ZDDTF EGDZW WZMSE GMSZM AHSER TRSED TYRSE GISEI GATWT HOZTY OFZWR AYRME YEERA YEMTR EATMA HMZYR EGTYO TDISE GEXTO SILEH ZXEIS TYOTY ISEHH AYOWE ISZHE OJPHM EGEYZ ISTYO TDYZI ISZGZ JOSRZ PZJSA KEAFF EHHIZ AYPHH BABEG HDGZX ISAIB AGIZD BAGTH TYISE MAGXA PLEIS EPVYE MMSAI MAHOZ TYOZY TAXSE ARTYO RZMYI ZIAWV IZAWD GERZO EGTSE MAHIS EREAW EGBEG JOOTA HSZME RISEX ZYAWT HAIZL AFVTY XAPLE SEVYZ MHHZX EISTY OALZJ IHAGA HDAXT WPTIH AWZYO HSZIL JIJYW EHHPZ JFZXE JBMTI SHZXE ISTYO DGZXI SEYAQ TMAGG EFZGR TITHA WWMES AKEAY PYEMH ZYISE NGAPS AGGP
--- /dev/null
+NEWJXVOIYZFLRFJINDFVEQAESOGFEZKMXECCIQCRNPZJTBEOJEPFSXVLNDOWXRTRTZBNLRPLJWS WXUKWOLQBIGJESRWEKBPXMPSRWRNMSEVVEGVMHTXLQCGZJCMKMZGMMPOIESQSYDHVTBPXMPGKEV TWHPFKEVPXMLHKLRVLJQHYEFDIJYRZWPQZJCSUEAFXMLHZAVNPGPQRYTJXYSSJWBHJNNSIZVUMY DZVWFQJYPBESJVLFEVVMFUYWPWRQPCVWJWEKBWXYSSZVVPWYCITXVQRXTQRRAGZJCPVGRTXFTBN LRPLJHWCPAGBYLDGINTFZEVVWRGQJOGRXVUJNPRNMGJQDHCIOGJMXLTKIEPSTYOEHCGVMLDJLRY MQWBFXEGXZCBKSZQVWZKUEAKIQTGFJGGRTFHUYEKRLEVVHNAWTHWKLYWGPXMVWPCTJHWCPAQXGP RZWPQZJCSUYAVMQEVVESVIWYCFRNPHNAZRRGQFJLPFEEFXMPHIEVPFDEVVRZAHTNIDIAVWFCSII NFCYZCKLRAATFZURBVFJLFRGYQWJTBJTREXNZBSYGKJNNOEWGGEQLPFEEFEKFZCXECMSEVVRVVL NYYKLRAANWZJYSHMHPHYIWQYWYSPXBVLJDHRXVQRBTZCFRFMKQWTYYVMSEVVFYCGPZIKFHVQDXO GMFEPJLFRRQVLJXOKGUGWNDHFPRYMQWUZZROIJYCLKUNMLSHKSPJIHVHYIJCCNHWCPUKHJEVZWQ KEWJIEHRTXMPPFEEFWNYHYIPGPQLFNLRTIXSSYEFNMAPRWSEVLJWOJXSGARZBKLFKJFYMKLVPKM LDGIAUXTXSZHBPSYHOEXBWVXECICSQVLZHKIARIWSOGWBPIILMZAVNPWPHLVACRICSKVVGZJTHZ
+JAQXFYRPSHCVJCSRHVPKYSWJXUGRBTGYQRNYHVGRVNJ
--- /dev/null
+samothrace
+paume
+musees
+musee
+valenay
+jeu
+montal
+montauban
+louvigny
+abbaye
+curated
+jaujard
+jahan
+loc
+albinguillot
+dieu
+reinstallation
+koblenz
+fonkenell
+vaux
+laure
+guillaume
+chambord
--- /dev/null
+1a: caesar_decipher(c1a, 8)
+1b: caesar_decipher(c1b, 14)
+2a: affine_decipher(c2a, 3, 3, True)
+2b: caesar_decipher(c2b, 6)
+3a: affine_decipher(c3a, 7, 8, True)
+# with open('2013/mona-lisa-words.txt') as f: mona_lisa_words = [line.rstrip() for line in f]
+# keyword_break(c4a, wordlist=mona_lisa_words)
+3b: keyword_decipher(c3b, 'louvigny', 2)
+4a: keyword_decipher(c4a, 'montal', 2)
+4b: keyword_decipher(c4b, 'salvation', 2)
+5a: keyword_decipher(c5a, 'alfredo', 2)
+5b: vigenere_decipher(c5bs, 'florence')
+
--- /dev/null
+import string
+import collections
+import norms
+import logging
+from itertools import zip_longest, cycle
+from segment import segment
+from multiprocessing import Pool
+
+from cipher import *
+
+# To time a run:
+#
+# import timeit
+# c5a = open('2012/5a.ciphertext', 'r').read()
+# timeit.timeit('keyword_break(c5a)', setup='gc.enable() ; from __main__ import c5a ; from cipher import keyword_break', number=1)
+# timeit.repeat('keyword_break_mp(c5a, chunksize=500)', setup='gc.enable() ; from __main__ import c5a ; from cipher import keyword_break_mp', repeat=5, number=1)
+
+
+english_counts = collections.defaultdict(int)
+with open('count_1l.txt', 'r') as f:
+ for line in f:
+ (letter, count) = line.split("\t")
+ english_counts[letter] = int(count)
+normalised_english_counts = norms.normalise(english_counts)
+
+english_bigram_counts = collections.defaultdict(int)
+with open('count_2l.txt', 'r') as f:
+ for line in f:
+ (bigram, count) = line.split("\t")
+ english_bigram_counts[bigram] = int(count)
+normalised_english_bigram_counts = norms.normalise(english_bigram_counts)
+
+english_trigram_counts = collections.defaultdict(int)
+with open('count_3l.txt', 'r') as f:
+ for line in f:
+ (trigram, count) = line.split("\t")
+ english_trigram_counts[trigram] = int(count)
+normalised_english_trigram_counts = norms.normalise(english_trigram_counts)
+
+
+with open('words.txt', 'r') as f:
+ keywords = [line.rstrip() for line in f]
+
+transpositions = collections.defaultdict(list)
+for word in keywords:
+ transpositions[transpositions_of(word)] += [word]
+
+def frequencies(text):
+ """Count the number of occurrences of each character in text
+
+ >>> sorted(frequencies('abcdefabc').items())
+ [('a', 2), ('b', 2), ('c', 2), ('d', 1), ('e', 1), ('f', 1)]
+ >>> sorted(frequencies('the quick brown fox jumped over the lazy ' \
+ 'dog').items()) # doctest: +NORMALIZE_WHITESPACE
+ [(' ', 8), ('a', 1), ('b', 1), ('c', 1), ('d', 2), ('e', 4), ('f', 1),
+ ('g', 1), ('h', 2), ('i', 1), ('j', 1), ('k', 1), ('l', 1), ('m', 1),
+ ('n', 1), ('o', 4), ('p', 1), ('q', 1), ('r', 2), ('t', 2), ('u', 2),
+ ('v', 1), ('w', 1), ('x', 1), ('y', 1), ('z', 1)]
+ >>> sorted(frequencies('The Quick BROWN fox jumped! over... the ' \
+ '(9lazy) DOG').items()) # doctest: +NORMALIZE_WHITESPACE
+ [(' ', 8), ('!', 1), ('(', 1), (')', 1), ('.', 3), ('9', 1), ('B', 1),
+ ('D', 1), ('G', 1), ('N', 1), ('O', 2), ('Q', 1), ('R', 1), ('T', 1),
+ ('W', 1), ('a', 1), ('c', 1), ('d', 1), ('e', 4), ('f', 1), ('h', 2),
+ ('i', 1), ('j', 1), ('k', 1), ('l', 1), ('m', 1), ('o', 2), ('p', 1),
+ ('r', 1), ('t', 1), ('u', 2), ('v', 1), ('x', 1), ('y', 1), ('z', 1)]
+ >>> sorted(frequencies(sanitise('The Quick BROWN fox jumped! over... ' \
+ 'the (9lazy) DOG')).items()) # doctest: +NORMALIZE_WHITESPACE
+ [('a', 1), ('b', 1), ('c', 1), ('d', 2), ('e', 4), ('f', 1), ('g', 1),
+ ('h', 2), ('i', 1), ('j', 1), ('k', 1), ('l', 1), ('m', 1), ('n', 1),
+ ('o', 4), ('p', 1), ('q', 1), ('r', 2), ('t', 2), ('u', 2), ('v', 1),
+ ('w', 1), ('x', 1), ('y', 1), ('z', 1)]
+ >>> frequencies('abcdefabcdef')['x']
+ 0
+ """
+ #counts = collections.defaultdict(int)
+ #for c in text:
+ # counts[c] += 1
+ #return counts
+ return collections.Counter(c for c in text)
+letter_frequencies = frequencies
+
+
+
+def caesar_break(message,
+ metric=norms.euclidean_distance,
+ target_counts=normalised_english_counts,
+ message_frequency_scaling=norms.normalise):
+ """Breaks a Caesar cipher using frequency analysis
+
+ >>> caesar_break('ibxcsyorsaqcheyklxivoexlevmrimwxsfiqevvmihrsasrxliwyrh' \
+ 'ecjsppsamrkwleppfmergefifvmhixscsymjcsyqeoixlm') # doctest: +ELLIPSIS
+ (4, 0.080345432737...)
+ >>> caesar_break('wxwmaxdgheetgwuxztgptedbgznitgwwhpguxyhkxbmhvvtlbhgtee' \
+ 'raxlmhiixweblmxgxwmhmaxybkbgztgwztsxwbgmxgmert') # doctest: +ELLIPSIS
+ (19, 0.11189290326...)
+ >>> caesar_break('yltbbqnqnzvguvaxurorgenafsbezqvagbnornfgsbevpnaabjurer' \
+ 'svaquvzyvxrnznazlybequrvfohgriraabjtbaruraprur') # doctest: +ELLIPSIS
+ (13, 0.08293968842...)
+ """
+ sanitised_message = sanitise(message)
+ best_shift = 0
+ best_fit = float("inf")
+ for shift in range(26):
+ plaintext = caesar_decipher(sanitised_message, shift)
+ counts = message_frequency_scaling(letter_frequencies(plaintext))
+ fit = metric(target_counts, counts)
+ logger.debug('Caesar break attempt using key {0} gives fit of {1} '
+ 'and decrypt starting: {2}'.format(shift, fit, plaintext[:50]))
+ if fit < best_fit:
+ best_fit = fit
+ best_shift = shift
+ logger.info('Caesar break best fit: key {0} gives fit of {1} and '
+ 'decrypt starting: {2}'.format(best_shift, best_fit,
+ caesar_decipher(sanitised_message, best_shift)[:50]))
+ return best_shift, best_fit
+
+def affine_break(message,
+ metric=norms.euclidean_distance,
+ target_counts=normalised_english_counts,
+ message_frequency_scaling=norms.normalise):
+ """Breaks an affine cipher using frequency analysis
+
+ >>> affine_break('lmyfu bkuusd dyfaxw claol psfaom jfasd snsfg jfaoe ls ' \
+ 'omytd jlaxe mh jm bfmibj umis hfsul axubafkjamx. ls kffkxwsd jls ' \
+ 'ofgbjmwfkiu olfmxmtmwaokttg jlsx ls kffkxwsd jlsi zg tsxwjl. jlsx ' \
+ 'ls umfjsd jlsi zg hfsqysxog. ls dmmdtsd mx jls bats mh bkbsf. ls ' \
+ 'bfmctsd kfmyxd jls lyj, mztanamyu xmc jm clm cku tmmeaxw kj lai kxd ' \
+ 'clm ckuxj.') # doctest: +ELLIPSIS
+ ((15, 22, True), 0.0598745365924...)
+ """
+ sanitised_message = sanitise(message)
+ best_multiplier = 0
+ best_adder = 0
+ best_one_based = True
+ best_fit = float("inf")
+ for one_based in [True, False]:
+ for multiplier in range(1, 26, 2):
+ for adder in range(26):
+ plaintext = affine_decipher(sanitised_message,
+ multiplier, adder, one_based)
+ counts = message_frequency_scaling(letter_frequencies(plaintext))
+ fit = metric(target_counts, counts)
+ logger.debug('Affine break attempt using key {0}x+{1} ({2}) '
+ 'gives fit of {3} and decrypt starting: {4}'.
+ format(multiplier, adder, one_based, fit,
+ plaintext[:50]))
+ if fit < best_fit:
+ best_fit = fit
+ best_multiplier = multiplier
+ best_adder = adder
+ best_one_based = one_based
+ logger.info('Affine break best fit with key {0}x+{1} ({2}) gives fit of {3} '
+ 'and decrypt starting: {4}'.format(
+ best_multiplier, best_adder, best_one_based, best_fit,
+ affine_decipher(sanitised_message, best_multiplier,
+ best_adder, best_one_based)[:50]))
+ return (best_multiplier, best_adder, best_one_based), best_fit
+
+def keyword_break(message,
+ wordlist=keywords,
+ metric=norms.euclidean_distance,
+ target_counts=normalised_english_counts,
+ message_frequency_scaling=norms.normalise):
+ """Breaks a keyword substitution cipher using a dictionary and
+ frequency analysis
+
+ >>> keyword_break(keyword_encipher('this is a test message for the ' \
+ 'keyword decipherment', 'elephant', 1), \
+ wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS
+ (('elephant', 1), 0.1066453448861...)
+ """
+ best_keyword = ''
+ best_wrap_alphabet = True
+ best_fit = float("inf")
+ for wrap_alphabet in range(3):
+ for keyword in wordlist:
+ plaintext = keyword_decipher(message, keyword, wrap_alphabet)
+ counts = message_frequency_scaling(letter_frequencies(plaintext))
+ fit = metric(target_counts, counts)
+ logger.debug('Keyword break attempt using key {0} (wrap={1}) '
+ 'gives fit of {2} and decrypt starting: {3}'.format(
+ keyword, wrap_alphabet, fit,
+ sanitise(plaintext)[:50]))
+ if fit < best_fit:
+ best_fit = fit
+ best_keyword = keyword
+ best_wrap_alphabet = wrap_alphabet
+ logger.info('Keyword break best fit with key {0} (wrap={1}) gives fit of '
+ '{2} and decrypt starting: {3}'.format(best_keyword,
+ best_wrap_alphabet, best_fit, sanitise(
+ keyword_decipher(message, best_keyword,
+ best_wrap_alphabet))[:50]))
+ return (best_keyword, best_wrap_alphabet), best_fit
+
+def keyword_break_mp(message,
+ wordlist=keywords,
+ metric=norms.euclidean_distance,
+ target_counts=normalised_english_counts,
+ message_frequency_scaling=norms.normalise,
+ chunksize=500):
+ """Breaks a keyword substitution cipher using a dictionary and
+ frequency analysis
+
+ >>> keyword_break_mp(keyword_encipher('this is a test message for the ' \
+ 'keyword decipherment', 'elephant', 1), \
+ wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS
+ (('elephant', 1), 0.106645344886...)
+ """
+ with Pool() as pool:
+ helper_args = [(message, word, wrap, metric, target_counts,
+ message_frequency_scaling)
+ for word in wordlist for wrap in range(3)]
+ # Gotcha: the helper function here needs to be defined at the top level
+ # (limitation of Pool.starmap)
+ breaks = pool.starmap(keyword_break_worker, helper_args, chunksize)
+ return min(breaks, key=lambda k: k[1])
+
+def keyword_break_worker(message, keyword, wrap_alphabet, metric, target_counts,
+ message_frequency_scaling):
+ plaintext = keyword_decipher(message, keyword, wrap_alphabet)
+ counts = message_frequency_scaling(letter_frequencies(plaintext))
+ fit = metric(target_counts, counts)
+ logger.debug('Keyword break attempt using key {0} (wrap={1}) gives fit of '
+ '{2} and decrypt starting: {3}'.format(keyword,
+ wrap_alphabet, fit, sanitise(plaintext)[:50]))
+ return (keyword, wrap_alphabet), fit
+
+def scytale_break(message,
+ metric=norms.euclidean_distance,
+ target_counts=normalised_english_bigram_counts,
+ message_frequency_scaling=norms.normalise):
+ """Breaks a Scytale cipher
+
+ >>> scytale_break('tfeulchtrtteehwahsdehneoifeayfsondmwpltmaoalhikotoere' \
+ 'dcweatehiplwxsnhooacgorrcrcraotohsgullasenylrendaianeplscdriioto' \
+ 'aek') # doctest: +ELLIPSIS
+ (6, 0.092599933059...)
+ """
+ best_key = 0
+ best_fit = float("inf")
+ ngram_length = len(next(iter(target_counts.keys())))
+ for key in range(1, 20):
+ if len(message) % key == 0:
+ plaintext = scytale_decipher(message, key)
+ counts = message_frequency_scaling(frequencies(
+ ngrams(sanitise(plaintext), ngram_length)))
+ fit = metric(target_counts, counts)
+ logger.debug('Scytale break attempt using key {0} gives fit of '
+ '{1} and decrypt starting: {2}'.format(key,
+ fit, sanitise(plaintext)[:50]))
+ if fit < best_fit:
+ best_fit = fit
+ best_key = key
+ logger.info('Scytale break best fit with key {0} gives fit of {1} and '
+ 'decrypt starting: {2}'.format(best_key, best_fit,
+ sanitise(scytale_decipher(message, best_key))[:50]))
+ return best_key, best_fit
+
+def column_transposition_break(message,
+ translist=transpositions,
+ metric=norms.euclidean_distance,
+ target_counts=normalised_english_bigram_counts,
+ message_frequency_scaling=norms.normalise):
+ """Breaks a column transposition cipher using a dictionary and
+ n-gram frequency analysis
+
+ >>> column_transposition_break(column_transposition_encipher(sanitise( \
+ "It is a truth universally acknowledged, that a single man in \
+ possession of a good fortune, must be in want of a wife. However \
+ little known the feelings or views of such a man may be on his \
+ first entering a neighbourhood, this truth is so well fixed in the \
+ minds of the surrounding families, that he is considered the \
+ rightful property of some one or other of their daughters."), \
+ 'encipher'), \
+ translist={(2, 0, 5, 3, 1, 4, 6): ['encipher'], \
+ (5, 0, 6, 1, 3, 4, 2): ['fourteen'], \
+ (6, 1, 0, 4, 5, 3, 2): ['keyword']}) # doctest: +ELLIPSIS
+ ((2, 0, 5, 3, 1, 4, 6), 0.0628106372...)
+ >>> column_transposition_break(column_transposition_encipher(sanitise( \
+ "It is a truth universally acknowledged, that a single man in \
+ possession of a good fortune, must be in want of a wife. However \
+ little known the feelings or views of such a man may be on his \
+ first entering a neighbourhood, this truth is so well fixed in the \
+ minds of the surrounding families, that he is considered the \
+ rightful property of some one or other of their daughters."), \
+ 'encipher'), \
+ translist={(2, 0, 5, 3, 1, 4, 6): ['encipher'], \
+ (5, 0, 6, 1, 3, 4, 2): ['fourteen'], \
+ (6, 1, 0, 4, 5, 3, 2): ['keyword']}, \
+ target_counts=normalised_english_trigram_counts) # doctest: +ELLIPSIS
+ ((2, 0, 5, 3, 1, 4, 6), 0.0592259560...)
+ """
+ best_transposition = ''
+ best_fit = float("inf")
+ ngram_length = len(next(iter(target_counts.keys())))
+ for transposition in translist.keys():
+ if len(message) % len(transposition) == 0:
+ plaintext = column_transposition_decipher(message, transposition)
+ counts = message_frequency_scaling(frequencies(
+ ngrams(sanitise(plaintext), ngram_length)))
+ fit = metric(target_counts, counts)
+ logger.debug('Column transposition break attempt using key {0} '
+ 'gives fit of {1} and decrypt starting: {2}'.format(
+ translist[transposition][0], fit,
+ sanitise(plaintext)[:50]))
+ if fit < best_fit:
+ best_fit = fit
+ best_transposition = transposition
+ logger.info('Column transposition break best fit with key {0} gives fit '
+ 'of {1} and decrypt starting: {2}'.format(
+ translist[best_transposition][0],
+ best_fit, sanitise(
+ column_transposition_decipher(message,
+ best_transposition))[:50]))
+ return best_transposition, best_fit
+
+
+def column_transposition_break_mp(message,
+ translist=transpositions,
+ metric=norms.euclidean_distance,
+ target_counts=normalised_english_bigram_counts,
+ message_frequency_scaling=norms.normalise,
+ chunksize=500):
+ """Breaks a column transposition cipher using a dictionary and
+ n-gram frequency analysis
+
+ >>> column_transposition_break_mp(column_transposition_encipher(sanitise( \
+ "It is a truth universally acknowledged, that a single man in \
+ possession of a good fortune, must be in want of a wife. However \
+ little known the feelings or views of such a man may be on his \
+ first entering a neighbourhood, this truth is so well fixed in the \
+ minds of the surrounding families, that he is considered the \
+ rightful property of some one or other of their daughters."), \
+ 'encipher'), \
+ translist={(2, 0, 5, 3, 1, 4, 6): ['encipher'], \
+ (5, 0, 6, 1, 3, 4, 2): ['fourteen'], \
+ (6, 1, 0, 4, 5, 3, 2): ['keyword']}) # doctest: +ELLIPSIS
+ ((2, 0, 5, 3, 1, 4, 6), 0.0628106372...)
+ >>> column_transposition_break_mp(column_transposition_encipher(sanitise( \
+ "It is a truth universally acknowledged, that a single man in \
+ possession of a good fortune, must be in want of a wife. However \
+ little known the feelings or views of such a man may be on his \
+ first entering a neighbourhood, this truth is so well fixed in the \
+ minds of the surrounding families, that he is considered the \
+ rightful property of some one or other of their daughters."), \
+ 'encipher'), \
+ translist={(2, 0, 5, 3, 1, 4, 6): ['encipher'], \
+ (5, 0, 6, 1, 3, 4, 2): ['fourteen'], \
+ (6, 1, 0, 4, 5, 3, 2): ['keyword']}, \
+ target_counts=normalised_english_trigram_counts) # doctest: +ELLIPSIS
+ ((2, 0, 5, 3, 1, 4, 6), 0.0592259560...)
+ """
+ ngram_length = len(next(iter(target_counts.keys())))
+ with Pool() as pool:
+ helper_args = [(message, trans, metric, target_counts, ngram_length,
+ message_frequency_scaling)
+ for trans in translist.keys()]
+ # Gotcha: the helper function here needs to be defined at the top level
+ # (limitation of Pool.starmap)
+ breaks = pool.starmap(column_transposition_break_worker, helper_args, chunksize)
+ return min(breaks, key=lambda k: k[1])
+
+def column_transposition_break_worker(message, transposition, metric, target_counts,
+ ngram_length, message_frequency_scaling):
+ plaintext = column_transposition_decipher(message, transposition)
+ counts = message_frequency_scaling(frequencies(
+ ngrams(sanitise(plaintext), ngram_length)))
+ fit = metric(target_counts, counts)
+ logger.debug('Column transposition break attempt using key {0} '
+ 'gives fit of {1} and decrypt starting: {2}'.format(
+ transposition, fit,
+ sanitise(plaintext)[:50]))
+ return transposition, fit
+
+def vigenere_keyword_break(message,
+ wordlist=keywords,
+ metric=norms.euclidean_distance,
+ target_counts=normalised_english_counts,
+ message_frequency_scaling=norms.normalise):
+ """Breaks a vigenere cipher using a dictionary and
+ frequency analysis
+
+ >>> vigenere_keyword_break(vigenere_encipher(sanitise('this is a test ' \
+ 'message for the vigenere decipherment'), 'cat'), \
+ wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS
+ ('cat', 0.15965224935...)
+ """
+ best_keyword = ''
+ best_fit = float("inf")
+ for keyword in wordlist:
+ plaintext = vigenere_decipher(message, keyword)
+ counts = message_frequency_scaling(letter_frequencies(plaintext))
+ fit = metric(target_counts, counts)
+ logger.debug('Vigenere break attempt using key {0} '
+ 'gives fit of {1} and decrypt starting: {2}'.format(
+ keyword, fit,
+ sanitise(plaintext)[:50]))
+ if fit < best_fit:
+ best_fit = fit
+ best_keyword = keyword
+ logger.info('Vigenere break best fit with key {0} gives fit '
+ 'of {1} and decrypt starting: {2}'.format(best_keyword,
+ best_fit, sanitise(
+ vigenere_decipher(message, best_keyword))[:50]))
+ return best_keyword, best_fit
+
+def vigenere_keyword_break_mp(message,
+ wordlist=keywords,
+ metric=norms.euclidean_distance,
+ target_counts=normalised_english_counts,
+ message_frequency_scaling=norms.normalise,
+ chunksize=500):
+ """Breaks a vigenere cipher using a dictionary and
+ frequency analysis
+
+ >>> vigenere_keyword_break_mp(vigenere_encipher(sanitise('this is a test ' \
+ 'message for the vigenere decipherment'), 'cat'), \
+ wordlist=['cat', 'elephant', 'kangaroo']) # doctest: +ELLIPSIS
+ ('cat', 0.159652249358...)
+ """
+ with Pool() as pool:
+ helper_args = [(message, word, metric, target_counts,
+ message_frequency_scaling)
+ for word in wordlist]
+ # Gotcha: the helper function here needs to be defined at the top level
+ # (limitation of Pool.starmap)
+ breaks = pool.starmap(vigenere_keyword_break_worker, helper_args, chunksize)
+ return min(breaks, key=lambda k: k[1])
+
+def vigenere_keyword_break_worker(message, keyword, metric, target_counts,
+ message_frequency_scaling):
+ plaintext = vigenere_decipher(message, keyword)
+ counts = message_frequency_scaling(letter_frequencies(plaintext))
+ fit = metric(target_counts, counts)
+ logger.debug('Vigenere keyword break attempt using key {0} gives fit of '
+ '{1} and decrypt starting: {2}'.format(keyword,
+ fit, sanitise(plaintext)[:50]))
+ return keyword, fit
+
+
+def vigenere_ic_break(message, target_counts=normalised_english_counts):
+ key_length = vigenere_key_length(message),
+ key = vigenere_find_key(message, key_length)
+ return key
+
+def vigenere_key_length(message):
+ best_length = 0
+ best_ic = 0.0
+ for trial_length in range(1, 20):
+ splits = every_nth(message, trial_length)
+ freqs = [norms.scale(frequencies(s)) for s in splits]
+ ic = sum([sum([f ** 2 for f in fs.values()]) for fs in freqs]) / trial_length
+ logger.debug('Vigenere key length of {0} gives IC of {1}'.
+ format(trial_length, ic))
+ if ic > best_ic:
+ best_length = trial_length
+ best_ic = ic
+ return best_length, best_ic
+
+def vigenere_find_key(message, key_length):
+ splits = every_nth(message, key_length)
+ return ''.join([chr(caesar_break(s)[0] + ord('a')) for s in splits])
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod()
+
--- /dev/null
+l1, normalised_english_counts, normalise, 300, 0.9992
+l1, normalised_english_counts, normalise, 100, 0.9996
+l1, normalised_english_counts, normalise, 50, 0.9992
+l1, normalised_english_counts, normalise, 30, 0.9914
+l1, normalised_english_counts, normalise, 20, 0.9532
+l1, normalised_english_counts, normalise, 10, 0.7442
+l1, normalised_english_counts, normalise, 5, 0.4358
+l1, normalised_english_counts, scale, 300, 1.0
+l1, normalised_english_counts, scale, 100, 0.999
+l1, normalised_english_counts, scale, 50, 0.9988
+l1, normalised_english_counts, scale, 30, 0.9848
+l1, normalised_english_counts, scale, 20, 0.9316
+l1, normalised_english_counts, scale, 10, 0.715
+l1, normalised_english_counts, scale, 5, 0.436
+l1, scaled_english_counts, normalise, 300, 0.9994
+l1, scaled_english_counts, normalise, 100, 0.9998
+l1, scaled_english_counts, normalise, 50, 0.999
+l1, scaled_english_counts, normalise, 30, 0.9868
+l1, scaled_english_counts, normalise, 20, 0.9482
+l1, scaled_english_counts, normalise, 10, 0.7434
+l1, scaled_english_counts, normalise, 5, 0.4532
+l1, scaled_english_counts, scale, 300, 0.9996
+l1, scaled_english_counts, scale, 100, 1.0
+l1, scaled_english_counts, scale, 50, 0.9988
+l1, scaled_english_counts, scale, 30, 0.9874
+l1, scaled_english_counts, scale, 20, 0.9488
+l1, scaled_english_counts, scale, 10, 0.745
+l1, scaled_english_counts, scale, 5, 0.4548
+l2, normalised_english_counts, normalise, 300, 0.9994
+l2, normalised_english_counts, normalise, 100, 0.9992
+l2, normalised_english_counts, normalise, 50, 0.9978
+l2, normalised_english_counts, normalise, 30, 0.9836
+l2, normalised_english_counts, normalise, 20, 0.9318
+l2, normalised_english_counts, normalise, 10, 0.7072
+l2, normalised_english_counts, normalise, 5, 0.4294
+l2, normalised_english_counts, scale, 300, 0.9988
+l2, normalised_english_counts, scale, 100, 0.9998
+l2, normalised_english_counts, scale, 50, 0.9978
+l2, normalised_english_counts, scale, 30, 0.9868
+l2, normalised_english_counts, scale, 20, 0.9364
+l2, normalised_english_counts, scale, 10, 0.7136
+l2, normalised_english_counts, scale, 5, 0.446
+l2, scaled_english_counts, normalise, 300, 0.9992
+l2, scaled_english_counts, normalise, 100, 0.9996
+l2, scaled_english_counts, normalise, 50, 0.9984
+l2, scaled_english_counts, normalise, 30, 0.9854
+l2, scaled_english_counts, normalise, 20, 0.9328
+l2, scaled_english_counts, normalise, 10, 0.7122
+l2, scaled_english_counts, normalise, 5, 0.4328
+l2, scaled_english_counts, scale, 300, 1.0
+l2, scaled_english_counts, scale, 100, 0.9998
+l2, scaled_english_counts, scale, 50, 0.9972
+l2, scaled_english_counts, scale, 30, 0.9842
+l2, scaled_english_counts, scale, 20, 0.9356
+l2, scaled_english_counts, scale, 10, 0.7126
+l2, scaled_english_counts, scale, 5, 0.4318
+l3, normalised_english_counts, normalise, 300, 0.9996
+l3, normalised_english_counts, normalise, 100, 0.999
+l3, normalised_english_counts, normalise, 50, 0.994
+l3, normalised_english_counts, normalise, 30, 0.9658
+l3, normalised_english_counts, normalise, 20, 0.8926
+l3, normalised_english_counts, normalise, 10, 0.6252
+l3, normalised_english_counts, normalise, 5, 0.3974
+l3, normalised_english_counts, scale, 300, 0.9996
+l3, normalised_english_counts, scale, 100, 0.998
+l3, normalised_english_counts, scale, 50, 0.9828
+l3, normalised_english_counts, scale, 30, 0.9334
+l3, normalised_english_counts, scale, 20, 0.8304
+l3, normalised_english_counts, scale, 10, 0.5968
+l3, normalised_english_counts, scale, 5, 0.4114
+l3, scaled_english_counts, normalise, 300, 0.9994
+l3, scaled_english_counts, normalise, 100, 0.9984
+l3, scaled_english_counts, normalise, 50, 0.9876
+l3, scaled_english_counts, normalise, 30, 0.9284
+l3, scaled_english_counts, normalise, 20, 0.8322
+l3, scaled_english_counts, normalise, 10, 0.579
+l3, scaled_english_counts, normalise, 5, 0.3466
+l3, scaled_english_counts, scale, 300, 1.0
+l3, scaled_english_counts, scale, 100, 0.999
+l3, scaled_english_counts, scale, 50, 0.994
+l3, scaled_english_counts, scale, 30, 0.9688
+l3, scaled_english_counts, scale, 20, 0.8952
+l3, scaled_english_counts, scale, 10, 0.6416
+l3, scaled_english_counts, scale, 5, 0.4042
+cosine_distance, normalised_english_counts, normalise, 300, 0.9994
+cosine_distance, normalised_english_counts, normalise, 100, 1.0
+cosine_distance, normalised_english_counts, normalise, 50, 0.9978
+cosine_distance, normalised_english_counts, normalise, 30, 0.9856
+cosine_distance, normalised_english_counts, normalise, 20, 0.9374
+cosine_distance, normalised_english_counts, normalise, 10, 0.7212
+cosine_distance, normalised_english_counts, normalise, 5, 0.4282
+cosine_distance, normalised_english_counts, scale, 300, 0.9998
+cosine_distance, normalised_english_counts, scale, 100, 0.9994
+cosine_distance, normalised_english_counts, scale, 50, 0.9972
+cosine_distance, normalised_english_counts, scale, 30, 0.9846
+cosine_distance, normalised_english_counts, scale, 20, 0.9324
+cosine_distance, normalised_english_counts, scale, 10, 0.7144
+cosine_distance, normalised_english_counts, scale, 5, 0.4284
+cosine_distance, scaled_english_counts, normalise, 300, 0.9994
+cosine_distance, scaled_english_counts, normalise, 100, 0.9996
+cosine_distance, scaled_english_counts, normalise, 50, 0.9978
+cosine_distance, scaled_english_counts, normalise, 30, 0.9856
+cosine_distance, scaled_english_counts, normalise, 20, 0.935
+cosine_distance, scaled_english_counts, normalise, 10, 0.7232
+cosine_distance, scaled_english_counts, normalise, 5, 0.415
+cosine_distance, scaled_english_counts, scale, 300, 0.9982
+cosine_distance, scaled_english_counts, scale, 100, 0.9988
+cosine_distance, scaled_english_counts, scale, 50, 0.9976
+cosine_distance, scaled_english_counts, scale, 30, 0.9844
+cosine_distance, scaled_english_counts, scale, 20, 0.9314
+cosine_distance, scaled_english_counts, scale, 10, 0.7102
+cosine_distance, scaled_english_counts, scale, 5, 0.4376
+harmonic_mean, normalised_english_counts, normalise, 300, 0.4684
+harmonic_mean, normalised_english_counts, normalise, 100, 0.5068
+harmonic_mean, normalised_english_counts, normalise, 50, 0.6978
+harmonic_mean, normalised_english_counts, normalise, 30, 0.593
+harmonic_mean, normalised_english_counts, normalise, 20, 0.536
+harmonic_mean, normalised_english_counts, normalise, 10, 0.4284
+harmonic_mean, normalised_english_counts, normalise, 5, 0.3542
+harmonic_mean, normalised_english_counts, scale, 300, 0.3602
+harmonic_mean, normalised_english_counts, scale, 100, 0.57
+harmonic_mean, normalised_english_counts, scale, 50, 0.795
+harmonic_mean, normalised_english_counts, scale, 30, 0.7694
+harmonic_mean, normalised_english_counts, scale, 20, 0.6924
+harmonic_mean, normalised_english_counts, scale, 10, 0.559
+harmonic_mean, normalised_english_counts, scale, 5, 0.39
+harmonic_mean, scaled_english_counts, normalise, 300, 0.1214
+harmonic_mean, scaled_english_counts, normalise, 100, 0.132
+harmonic_mean, scaled_english_counts, normalise, 50, 0.1956
+harmonic_mean, scaled_english_counts, normalise, 30, 0.2686
+harmonic_mean, scaled_english_counts, normalise, 20, 0.258
+harmonic_mean, scaled_english_counts, normalise, 10, 0.2042
+harmonic_mean, scaled_english_counts, normalise, 5, 0.227
+harmonic_mean, scaled_english_counts, scale, 300, 0.7956
+harmonic_mean, scaled_english_counts, scale, 100, 0.5672
+harmonic_mean, scaled_english_counts, scale, 50, 0.4404
+harmonic_mean, scaled_english_counts, scale, 30, 0.3584
+harmonic_mean, scaled_english_counts, scale, 20, 0.3012
+harmonic_mean, scaled_english_counts, scale, 10, 0.2136
+harmonic_mean, scaled_english_counts, scale, 5, 0.1426
+geometric_mean, normalised_english_counts, normalise, 300, 0.9996
+geometric_mean, normalised_english_counts, normalise, 100, 0.9992
+geometric_mean, normalised_english_counts, normalise, 50, 0.9928
+geometric_mean, normalised_english_counts, normalise, 30, 0.9552
+geometric_mean, normalised_english_counts, normalise, 20, 0.8936
+geometric_mean, normalised_english_counts, normalise, 10, 0.6582
+geometric_mean, normalised_english_counts, normalise, 5, 0.4316
+geometric_mean, normalised_english_counts, scale, 300, 0.97
+geometric_mean, normalised_english_counts, scale, 100, 0.9762
+geometric_mean, normalised_english_counts, scale, 50, 0.9724
+geometric_mean, normalised_english_counts, scale, 30, 0.9224
+geometric_mean, normalised_english_counts, scale, 20, 0.8496
+geometric_mean, normalised_english_counts, scale, 10, 0.6846
+geometric_mean, normalised_english_counts, scale, 5, 0.4268
+geometric_mean, scaled_english_counts, normalise, 300, 0.9556
+geometric_mean, scaled_english_counts, normalise, 100, 0.8724
+geometric_mean, scaled_english_counts, normalise, 50, 0.7176
+geometric_mean, scaled_english_counts, normalise, 30, 0.6536
+geometric_mean, scaled_english_counts, normalise, 20, 0.5586
+geometric_mean, scaled_english_counts, normalise, 10, 0.3926
+geometric_mean, scaled_english_counts, normalise, 5, 0.319
+geometric_mean, scaled_english_counts, scale, 300, 0.7822
+geometric_mean, scaled_english_counts, scale, 100, 0.5784
+geometric_mean, scaled_english_counts, scale, 50, 0.4318
+geometric_mean, scaled_english_counts, scale, 30, 0.349
+geometric_mean, scaled_english_counts, scale, 20, 0.2932
+geometric_mean, scaled_english_counts, scale, 10, 0.2098
+geometric_mean, scaled_english_counts, scale, 5, 0.1406
import string
import collections
+import logging
+from itertools import zip_longest, cycle
-english_counts = collections.defaultdict(int)
-with open('count_1l.txt', 'r') as f:
- for line in f:
- (letter, count) = line.split("\t")
- english_counts[letter] = int(count)
-modular_division = [[0]* 26 for i in range(26)]
-for i in range(26):
- for j in range(26):
- t = (i*j) % 26
- # therefore, i = t / j
- modular_division[t][j] = i
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.FileHandler('cipher.log'))
+logger.setLevel(logging.WARNING)
+#logger.setLevel(logging.INFO)
+#logger.setLevel(logging.DEBUG)
+modular_division_table = [[0]*26 for x in range(26)]
+for a in range(26):
+ for b in range(26):
+ c = (a * b) % 26
+ modular_division_table[b][c] = a
+
+def letters(text):
+ """Remove all non-alphabetic characters from a text
+ >>> letters('The Quick')
+ 'TheQuick'
+ >>> letters('The Quick BROWN fox jumped! over... the (9lazy) DOG')
+ 'TheQuickBROWNfoxjumpedoverthelazyDOG'
+ """
+ return ''.join([c for c in text if c in string.ascii_letters])
+
def sanitise(text):
- sanitised = [c.lower() for c in text if c in string.ascii_letters]
- return ''.join(sanitised)
-
-def letter_frequencies(message):
- frequencies = collections.defaultdict(int)
- for letter in sanitise(message):
- frequencies[letter]+=1
- return frequencies
-
-def scale_freq(frequencies):
- total= sum(frequencies.values())
- scaled_frequencies = collections.defaultdict(int)
- for letter in frequencies.keys():
- scaled_frequencies[letter] = frequencies[letter] / total
- return scaled_frequencies
-
-def value_diff(frequencies1, frequencies2):
- total= 0
- for letter in frequencies1.keys():
- total += abs(frequencies1[letter]-frequencies2[letter])
- return total
-
+ """Remove all non-alphabetic characters and convert the text to lowercase
+
+ >>> sanitise('The Quick')
+ 'thequick'
+ >>> sanitise('The Quick BROWN fox jumped! over... the (9lazy) DOG')
+ 'thequickbrownfoxjumpedoverthelazydog'
+ """
+ # sanitised = [c.lower() for c in text if c in string.ascii_letters]
+ # return ''.join(sanitised)
+ return letters(text).lower()
+
+def ngrams(text, n):
+ """Returns all n-grams of a text
+
+ >>> ngrams(sanitise('the quick brown fox'), 2) # doctest: +NORMALIZE_WHITESPACE
+ ['th', 'he', 'eq', 'qu', 'ui', 'ic', 'ck', 'kb', 'br', 'ro', 'ow', 'wn',
+ 'nf', 'fo', 'ox']
+ >>> ngrams(sanitise('the quick brown fox'), 4) # doctest: +NORMALIZE_WHITESPACE
+ ['theq', 'hequ', 'equi', 'quic', 'uick', 'ickb', 'ckbr', 'kbro', 'brow',
+ 'rown', 'ownf', 'wnfo', 'nfox']
+ """
+ return [text[i:i+n] for i in range(len(text)-n+1)]
+
+def every_nth(text, n, fillvalue=''):
+ """Returns n strings, each of which consists of every nth character,
+ starting with the 0th, 1st, 2nd, ... (n-1)th character
+
+ >>> every_nth(string.ascii_lowercase, 5)
+ ['afkpuz', 'bglqv', 'chmrw', 'dinsx', 'ejoty']
+ >>> every_nth(string.ascii_lowercase, 1)
+ ['abcdefghijklmnopqrstuvwxyz']
+ >>> every_nth(string.ascii_lowercase, 26) # doctest: +NORMALIZE_WHITESPACE
+ ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
+ 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
+ >>> every_nth(string.ascii_lowercase, 5, fillvalue='!')
+ ['afkpuz', 'bglqv!', 'chmrw!', 'dinsx!', 'ejoty!']
+ """
+ split_text = [text[i:i+n] for i in range(0, len(text), n)]
+ return [''.join(l) for l in zip_longest(*split_text, fillvalue=fillvalue)]
+
+def combine_every_nth(split_text):
+ """Reforms a text split into every_nth strings
+ >>> combine_every_nth(every_nth(string.ascii_lowercase, 5))
+ 'abcdefghijklmnopqrstuvwxyz'
+ >>> combine_every_nth(every_nth(string.ascii_lowercase, 1))
+ 'abcdefghijklmnopqrstuvwxyz'
+ >>> combine_every_nth(every_nth(string.ascii_lowercase, 26))
+ 'abcdefghijklmnopqrstuvwxyz'
+ """
+ return ''.join([''.join(l)
+ for l in zip_longest(*split_text, fillvalue='')])
+
+def transpose(items, transposition):
+ """Moves items around according to the given transposition
+
+ >>> transpose(['a', 'b', 'c', 'd'], (0,1,2,3))
+ ['a', 'b', 'c', 'd']
+ >>> transpose(['a', 'b', 'c', 'd'], (3,1,2,0))
+ ['d', 'b', 'c', 'a']
+ >>> transpose([10,11,12,13,14,15], (3,2,4,1,5,0))
+ [13, 12, 14, 11, 15, 10]
+ """
+ transposed = [''] * len(transposition)
+ for p, t in enumerate(transposition):
+ transposed[p] = items[t]
+ return transposed
+
+def untranspose(items, transposition):
+ """Undoes a transpose
+
+ >>> untranspose(['a', 'b', 'c', 'd'], [0,1,2,3])
+ ['a', 'b', 'c', 'd']
+ >>> untranspose(['d', 'b', 'c', 'a'], [3,1,2,0])
+ ['a', 'b', 'c', 'd']
+ >>> untranspose([13, 12, 14, 11, 15, 10], [3,2,4,1,5,0])
+ [10, 11, 12, 13, 14, 15]
+ """
+ transposed = [''] * len(transposition)
+ for p, t in enumerate(transposition):
+ transposed[t] = items[p]
+ return transposed
+
+
+
+def deduplicate(text):
+ return list(collections.OrderedDict.fromkeys(text))
-def caesar_cipher_letter(letter, shift):
+
+
+def caesar_encipher_letter(letter, shift):
+ """Encipher a letter, given a shift amount
+
+ >>> caesar_encipher_letter('a', 1)
+ 'b'
+ >>> caesar_encipher_letter('a', 2)
+ 'c'
+ >>> caesar_encipher_letter('b', 2)
+ 'd'
+ >>> caesar_encipher_letter('x', 2)
+ 'z'
+ >>> caesar_encipher_letter('y', 2)
+ 'a'
+ >>> caesar_encipher_letter('z', 2)
+ 'b'
+ >>> caesar_encipher_letter('z', -1)
+ 'y'
+ >>> caesar_encipher_letter('a', -1)
+ 'z'
+ """
if letter in string.ascii_letters:
- if letter in string.ascii_lowercase:
- return chr((ord(letter) - ord('a') + shift) % 26 + ord('a'))
+ if letter in string.ascii_uppercase:
+ alphabet_start = ord('A')
else:
- new_letter = letter.lower()
- yolo = chr((ord(new_letter) - ord('a') + shift) % 26 + ord('a'))
- return yolo.upper()
+ alphabet_start = ord('a')
+ return chr(((ord(letter) - alphabet_start + shift) % 26) +
+ alphabet_start)
else:
return letter
def caesar_decipher_letter(letter, shift):
- return caesar_cipher_letter(letter, -shift)
+ """Decipher a letter, given a shift amount
+
+ >>> caesar_decipher_letter('b', 1)
+ 'a'
+ >>> caesar_decipher_letter('b', 2)
+ 'z'
+ """
+ return caesar_encipher_letter(letter, -shift)
-def caesar_cipher_message(message, shift):
- big_cipher = [caesar_cipher_letter(l, shift) for l in message]
- return ''.join(big_cipher)
+def caesar_encipher(message, shift):
+ """Encipher a message with the Caesar cipher of given shift
+
+ >>> caesar_encipher('abc', 1)
+ 'bcd'
+ >>> caesar_encipher('abc', 2)
+ 'cde'
+ >>> caesar_encipher('abcxyz', 2)
+ 'cdezab'
+ >>> caesar_encipher('ab cx yz', 2)
+ 'cd ez ab'
+ """
+ enciphered = [caesar_encipher_letter(l, shift) for l in message]
+ return ''.join(enciphered)
-def caesar_decipher_message(message, shift):
- return caesar_cipher_message(message, -shift)
+def caesar_decipher(message, shift):
+ """Encipher a message with the Caesar cipher of given shift
+
+ >>> caesar_decipher('bcd', 1)
+ 'abc'
+ >>> caesar_decipher('cde', 2)
+ 'abc'
+ >>> caesar_decipher('cd ez ab', 2)
+ 'ab cx yz'
+ """
+ return caesar_encipher(message, -shift)
-def affine_cipher_letter(letter, multiplier, shift, one_based=True):
+def affine_encipher_letter(letter, multiplier=1, adder=0, one_based=True):
+ """Encipher a letter, given a multiplier and adder
+
+ >>> ''.join([affine_encipher_letter(l, 3, 5, True) \
+ for l in string.ascii_uppercase])
+ 'HKNQTWZCFILORUXADGJMPSVYBE'
+ >>> ''.join([affine_encipher_letter(l, 3, 5, False) \
+ for l in string.ascii_uppercase])
+ 'FILORUXADGJMPSVYBEHKNQTWZC'
+ """
if letter in string.ascii_letters:
- if letter in string.ascii_lowercase:
- alphastart = ord('a')
+ if letter in string.ascii_uppercase:
+ alphabet_start = ord('A')
else:
- alphastart = ord('A')
- letter_number = ord(letter) - alphastart
- if one_based: letter_number += 1
- enciphered_letter_number = letter_number * multiplier + shift
- if one_based: enciphered_letter_number -=1
- enciphered_letter = chr(enciphered_letter_number % 26 + alphastart)
- return enciphered_letter
+ alphabet_start = ord('a')
+ letter_number = ord(letter) - alphabet_start
+ if one_based: letter_number += 1
+ cipher_number = (letter_number * multiplier + adder) % 26
+ if one_based: cipher_number -= 1
+ return chr(cipher_number % 26 + alphabet_start)
else:
return letter
-
-def affine_decipher_letter(letter, multiplier, shift, one_based=True):
+
+def affine_decipher_letter(letter, multiplier=1, adder=0, one_based=True):
+ """Encipher a letter, given a multiplier and adder
+
+ >>> ''.join([affine_decipher_letter(l, 3, 5, True) \
+ for l in 'HKNQTWZCFILORUXADGJMPSVYBE'])
+ 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+ >>> ''.join([affine_decipher_letter(l, 3, 5, False) \
+ for l in 'FILORUXADGJMPSVYBEHKNQTWZC'])
+ 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+ """
if letter in string.ascii_letters:
- if letter in string.ascii_lowercase:
- alphastart = ord('a')
+ if letter in string.ascii_uppercase:
+ alphabet_start = ord('A')
else:
- alphastart = ord('A')
- letter_number = ord(letter) - alphastart
- if one_based: letter_number +=1
- after_unshift = letter_number - shift
- deciphered_letter_number = modular_division[after_unshift % 26][multiplier]
- if one_based: deciphered_letter_number -=1
- deciphered_letter = chr(deciphered_letter_number % 26 + alphastart)
- return deciphered_letter
+ alphabet_start = ord('a')
+ cipher_number = ord(letter) - alphabet_start
+ if one_based: cipher_number += 1
+ plaintext_number = ( modular_division_table[multiplier]
+ [(cipher_number - adder) % 26] )
+ if one_based: plaintext_number -= 1
+ return chr(plaintext_number % 26 + alphabet_start)
else:
return letter
+
+def affine_encipher(message, multiplier=1, adder=0, one_based=True):
+ """Encipher a message
+
+ >>> affine_encipher('hours passed during which jerico tried every ' \
+ 'trick he could think of', 15, 22, True)
+ 'lmyfu bkuusd dyfaxw claol psfaom jfasd snsfg jfaoe ls omytd jlaxe mh'
+ """
+ enciphered = [affine_encipher_letter(l, multiplier, adder, one_based)
+ for l in message]
+ return ''.join(enciphered)
+
+def affine_decipher(message, multiplier=1, adder=0, one_based=True):
+ """Decipher a message
+
+ >>> affine_decipher('lmyfu bkuusd dyfaxw claol psfaom jfasd snsfg ' \
+ 'jfaoe ls omytd jlaxe mh', 15, 22, True)
+ 'hours passed during which jerico tried every trick he could think of'
+ """
+ enciphered = [affine_decipher_letter(l, multiplier, adder, one_based)
+ for l in message]
+ return ''.join(enciphered)
+
+
+def keyword_cipher_alphabet_of(keyword, wrap_alphabet=0):
+ """Find the cipher alphabet given a keyword.
+ wrap_alphabet controls how the rest of the alphabet is added
+ after the keyword.
+ 0 : from 'a'
+ 1 : from the last letter in the sanitised keyword
+ 2 : from the largest letter in the sanitised keyword
+
+ >>> keyword_cipher_alphabet_of('bayes')
+ 'bayescdfghijklmnopqrtuvwxz'
+ >>> keyword_cipher_alphabet_of('bayes', 0)
+ 'bayescdfghijklmnopqrtuvwxz'
+ >>> keyword_cipher_alphabet_of('bayes', 1)
+ 'bayestuvwxzcdfghijklmnopqr'
+ >>> keyword_cipher_alphabet_of('bayes', 2)
+ 'bayeszcdfghijklmnopqrtuvwx'
+ """
+ if wrap_alphabet == 0:
+ cipher_alphabet = ''.join(deduplicate(sanitise(keyword) +
+ string.ascii_lowercase))
+ else:
+ if wrap_alphabet == 1:
+ last_keyword_letter = deduplicate(sanitise(keyword))[-1]
+ else:
+ last_keyword_letter = sorted(sanitise(keyword))[-1]
+ last_keyword_position = string.ascii_lowercase.find(
+ last_keyword_letter) + 1
+ cipher_alphabet = ''.join(
+ deduplicate(sanitise(keyword) +
+ string.ascii_lowercase[last_keyword_position:] +
+ string.ascii_lowercase))
+ return cipher_alphabet
+
+
+def keyword_encipher(message, keyword, wrap_alphabet=0):
+ """Enciphers a message with a keyword substitution cipher.
+ wrap_alphabet controls how the rest of the alphabet is added
+ after the keyword.
+ 0 : from 'a'
+ 1 : from the last letter in the sanitised keyword
+ 2 : from the largest letter in the sanitised keyword
+
+ >>> keyword_encipher('test message', 'bayes')
+ 'rsqr ksqqbds'
+ >>> keyword_encipher('test message', 'bayes', 0)
+ 'rsqr ksqqbds'
+ >>> keyword_encipher('test message', 'bayes', 1)
+ 'lskl dskkbus'
+ >>> keyword_encipher('test message', 'bayes', 2)
+ 'qspq jsppbcs'
+ """
+ cipher_alphabet = keyword_cipher_alphabet_of(keyword, wrap_alphabet)
+ cipher_translation = ''.maketrans(string.ascii_lowercase, cipher_alphabet)
+ return message.lower().translate(cipher_translation)
+
+def keyword_decipher(message, keyword, wrap_alphabet=0):
+ """Deciphers a message with a keyword substitution cipher.
+ wrap_alphabet controls how the rest of the alphabet is added
+ after the keyword.
+ 0 : from 'a'
+ 1 : from the last letter in the sanitised keyword
+ 2 : from the largest letter in the sanitised keyword
-def affine_cipher_message(message, multiplier, shift, one_based=True):
- big_cipher = [affine_cipher_letter(l, multiplier, shift, one_based) for l in message]
- return ''.join(big_cipher)
-
-def affine_decipher_message(message, multiplier, shift, one_based=True):
- big_decipher = [affine_decipher_letter(l, multiplier, shift, one_based) for l in message]
- return ''.join(big_decipher)
-
-
-def caesar_break(message):
- best_key = 0
- best_fit = float("inf")
- for shift in range(26):
- plaintxt = caesar_decipher_message(message, shift)
- lettertxt = letter_frequencies(plaintxt)
- total1 = scale_freq(lettertxt)
- total2 = scale_freq(english_counts)
- fit = value_diff(total2, total1)
- if fit < best_fit:
- best_key = shift
- best_fit = fit
- return best_key
-
-def affine_break(message):
- best_key = (0, 0, 0)
- best_fit = float("inf")
- for multiplier in range(1, 26, 2):
- for shift in range(26):
- for one_based in [True, False]:
- plaintxt = affine_decipher_message(message, multiplier, shift, one_based)
- lettertxt = letter_frequencies(plaintxt)
- total1 = scale_freq(lettertxt)
- total2 = scale_freq(english_counts)
- fit = value_diff(total2, total1)
- if fit < best_fit:
- best_key = (multiplier, shift, one_based)
- best_fit = fit
- return best_key
+ >>> keyword_decipher('rsqr ksqqbds', 'bayes')
+ 'test message'
+ >>> keyword_decipher('rsqr ksqqbds', 'bayes', 0)
+ 'test message'
+ >>> keyword_decipher('lskl dskkbus', 'bayes', 1)
+ 'test message'
+ >>> keyword_decipher('qspq jsppbcs', 'bayes', 2)
+ 'test message'
+ """
+ cipher_alphabet = keyword_cipher_alphabet_of(keyword, wrap_alphabet)
+ cipher_translation = ''.maketrans(cipher_alphabet, string.ascii_lowercase)
+ return message.lower().translate(cipher_translation)
+
+def scytale_encipher(message, rows):
+ """Enciphers using the scytale transposition cipher.
+ Message is padded with spaces to allow all rows to be the same length.
+
+ >>> scytale_encipher('thequickbrownfox', 3)
+ 'tcnhkfeboqrxuo iw '
+ >>> scytale_encipher('thequickbrownfox', 4)
+ 'tubnhirfecooqkwx'
+ >>> scytale_encipher('thequickbrownfox', 5)
+ 'tubn hirf ecoo qkwx '
+ >>> scytale_encipher('thequickbrownfox', 6)
+ 'tqcrnxhukof eibwo '
+ >>> scytale_encipher('thequickbrownfox', 7)
+ 'tqcrnx hukof eibwo '
+ """
+ if len(message) % rows != 0:
+ message += ' '*(rows - len(message) % rows)
+ row_length = round(len(message) / rows)
+ slices = [message[i:i+row_length]
+ for i in range(0, len(message), row_length)]
+ return ''.join([''.join(r) for r in zip_longest(*slices, fillvalue='')])
+
+def scytale_decipher(message, rows):
+ """Deciphers using the scytale transposition cipher.
+ Assumes the message is padded so that all rows are the same length.
+
+ >>> scytale_decipher('tcnhkfeboqrxuo iw ', 3)
+ 'thequickbrownfox '
+ >>> scytale_decipher('tubnhirfecooqkwx', 4)
+ 'thequickbrownfox'
+ >>> scytale_decipher('tubn hirf ecoo qkwx ', 5)
+ 'thequickbrownfox '
+ >>> scytale_decipher('tqcrnxhukof eibwo ', 6)
+ 'thequickbrownfox '
+ >>> scytale_decipher('tqcrnx hukof eibwo ', 7)
+ 'thequickbrownfox '
+ """
+ cols = round(len(message) / rows)
+ columns = [message[i:i+rows] for i in range(0, cols * rows, rows)]
+ return ''.join([''.join(c) for c in zip_longest(*columns, fillvalue='')])
+
+
+def transpositions_of(keyword):
+ """Finds the transpostions given by a keyword. For instance, the keyword
+ 'clever' rearranges to 'celrv', so the first column (0) stays first, the
+ second column (1) moves to third, the third column (2) moves to second,
+ and so on.
+
+ If passed a tuple, assume it's already a transposition and just return it.
+
+ >>> transpositions_of('clever')
+ (0, 2, 1, 4, 3)
+ >>> transpositions_of('fred')
+ (3, 2, 0, 1)
+ >>> transpositions_of((3, 2, 0, 1))
+ (3, 2, 0, 1)
+ """
+ if isinstance(keyword, tuple):
+ return keyword
+ else:
+ key = deduplicate(keyword)
+ transpositions = tuple(key.index(l) for l in sorted(key))
+ return transpositions
+
+def column_transposition_encipher(message, keyword, fillvalue=' '):
+ """Enciphers using the column transposition cipher.
+ Message is padded to allow all rows to be the same length.
+
+ >>> column_transposition_encipher('hellothere', 'clever')
+ 'hleolteher'
+ >>> column_transposition_encipher('hellothere', 'cleverly', fillvalue='!')
+ 'hleolthre!e!'
+ """
+ return column_transposition_worker(message, keyword, encipher=True,
+ fillvalue=fillvalue)
+
+def column_transposition_decipher(message, keyword, fillvalue=' '):
+ """Deciphers using the column transposition cipher.
+ Message is padded to allow all rows to be the same length.
+
+ >>> column_transposition_decipher('hleolteher', 'clever')
+ 'hellothere'
+ >>> column_transposition_decipher('hleolthre!e!', 'cleverly', fillvalue='?')
+ 'hellothere!!'
+ """
+ return column_transposition_worker(message, keyword, encipher=False,
+ fillvalue=fillvalue)
+
+def column_transposition_worker(message, keyword,
+ encipher=True, fillvalue=' '):
+ """Does the actual work of the column transposition cipher.
+ Message is padded with spaces to allow all rows to be the same length.
+
+ >>> column_transposition_worker('hellothere', 'clever')
+ 'hleolteher'
+ >>> column_transposition_worker('hellothere', 'clever', encipher=True)
+ 'hleolteher'
+ >>> column_transposition_worker('hleolteher', 'clever', encipher=False)
+ 'hellothere'
+ """
+ transpositions = transpositions_of(keyword)
+ columns = every_nth(message, len(transpositions), fillvalue=fillvalue)
+ if encipher:
+ transposed_columns = transpose(columns, transpositions)
+ else:
+ transposed_columns = untranspose(columns, transpositions)
+ return combine_every_nth(transposed_columns)
+
+def vigenere_encipher(message, keyword):
+ """Vigenere encipher
+
+ >>> vigenere_encipher('hello', 'abc')
+ 'hfnlp'
+ """
+ shifts = [ord(l) - ord('a') for l in sanitise(keyword)]
+ pairs = zip(message, cycle(shifts))
+ return ''.join([caesar_encipher_letter(l, k) for l, k in pairs])
+
+def vigenere_decipher(message, keyword):
+ """Vigenere decipher
+
+ >>> vigenere_decipher('hfnlp', 'abc')
+ 'hello'
+ """
+ shifts = [ord(l) - ord('a') for l in sanitise(keyword)]
+ pairs = zip(message, cycle(shifts))
+ return ''.join([caesar_decipher_letter(l, k) for l, k in pairs])
+
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod()
--- /dev/null
+import random
+from cipher import *
+
+
+corpus = sanitise(''.join([open('shakespeare.txt', 'r').read(), open('sherlock-holmes.txt', 'r').read(), open('war-and-peace.txt', 'r').read()]))
+corpus_length = len(corpus)
+
+scaled_english_counts = norms.scale(english_counts)
+
+
+metrics = [norms.l1, norms.l2, norms.l3, norms.cosine_distance, norms.harmonic_mean, norms.geometric_mean]
+corpus_frequencies = [normalised_english_counts, scaled_english_counts]
+scalings = [norms.normalise, norms.scale]
+message_lengths = [300, 100, 50, 30, 20, 10, 5]
+
+metric_names = ['l1', 'l2', 'l3', 'cosine_distance', 'harmonic_mean', 'geometric_mean']
+corpus_frequency_names = ['normalised_english_counts', 'scaled_english_counts']
+scaling_names = ['normalise', 'scale']
+
+trials = 5000
+
+scores = collections.defaultdict(int)
+for metric in range(len(metrics)):
+ scores[metric_names[metric]] = collections.defaultdict(int)
+ for corpus_freqency in range(len(corpus_frequencies)):
+ scores[metric_names[metric]][corpus_frequency_names[corpus_freqency]] = collections.defaultdict(int)
+ for scaling in range(len(scalings)):
+ scores[metric_names[metric]][corpus_frequency_names[corpus_freqency]][scaling_names[scaling]] = collections.defaultdict(int)
+ for message_length in message_lengths:
+ for i in range(trials):
+ sample_start = random.randint(0, corpus_length - message_length)
+ sample = corpus[sample_start:(sample_start + message_length)]
+ key = random.randint(1, 25)
+ sample_ciphertext = caesar_encipher(sample, key)
+ (found_key, score) = caesar_break(sample_ciphertext,
+ metric=metrics[metric],
+ target_frequencies=corpus_frequencies[corpus_freqency],
+ message_frequency_scaling=scalings[scaling])
+ if found_key == key:
+ scores[metric_names[metric]][corpus_frequency_names[corpus_freqency]][scaling_names[scaling]][message_length] += 1
+ print(', '.join([metric_names[metric],
+ corpus_frequency_names[corpus_freqency],
+ scaling_names[scaling],
+ str(message_length),
+ str(scores[metric_names[metric]][corpus_frequency_names[corpus_freqency]][scaling_names[scaling]][message_length] / trials) ]))
+
+
+with open('caesar_break_parameter_trials.csv', 'w') as f:
+ for metric in range(len(metrics)):
+ for corpus_freqency in range(len(corpus_frequencies)):
+ for scaling in range(len(scalings)):
+ for message_length in message_lengths:
+ print(', '.join([metric_names[metric],
+ corpus_frequency_names[corpus_freqency],
+ scaling_names[scaling],
+ str(message_length),
+ str(scores[metric_names[metric]][corpus_frequency_names[corpus_freqency]][scaling_names[scaling]][message_length] / trials) ]),
+ file=f)
+
+
\ No newline at end of file
--- /dev/null
+import collections
+import string
+
+def sanitise(text):
+ return [l.lower() for l in text if l in string.ascii_letters]
+
+corpora = ['shakespeare.txt', 'sherlock-holmes.txt', 'war-and-peace.txt']
+counts = collections.defaultdict(int)
+
+for corpus in corpora:
+ text = sanitise(open(corpus, 'r').read())
+ for letter in text:
+ counts[letter] += 1
+
+sorted_letters = sorted(counts, key=counts.get, reverse=True)
+
+with open('count_1l.txt', 'w') as f:
+ for l in sorted_letters:
+ f.write("{0}\t{1}\n".format(l, counts[l]))
+
+
\ No newline at end of file
--- /dev/null
+import cipher
+
+american = set(open('/usr/share/dict/american-english', 'r').readlines())
+british = set(open('/usr/share/dict/british-english', 'r').readlines())
+cracklib = set(open('/usr/share/dict/cracklib-small', 'r').readlines())
+
+words = american | british | cracklib
+
+sanitised_words = set()
+
+for w in words:
+ sanitised_words.add(cipher.sanitise(w))
+
+sanitised_words.discard('')
+
+with open('words.txt', 'w') as f:
+ f.write('\n'.join(sorted(sanitised_words, key=lambda w: (len(w), w))))
+ #for w in sanitised_words:
+ #f.write('{0}\n'.format(w))
+
+
+
+
+
+
+
+
--- /dev/null
+import collections
+
+def normalise(frequencies):
+ """Scale a set of frequencies so they sum to one
+
+ >>> sorted(normalise({1: 1, 2: 0}).items())
+ [(1, 1.0), (2, 0.0)]
+ >>> sorted(normalise({1: 1, 2: 1}).items())
+ [(1, 0.5), (2, 0.5)]
+ >>> sorted(normalise({1: 1, 2: 1, 3: 1}).items()) # doctest: +ELLIPSIS
+ [(1, 0.333...), (2, 0.333...), (3, 0.333...)]
+ >>> sorted(normalise({1: 1, 2: 2, 3: 1}).items())
+ [(1, 0.25), (2, 0.5), (3, 0.25)]
+ """
+ length = sum([f for f in frequencies.values()])
+ return collections.defaultdict(int, ((k, v / length)
+ for (k, v) in frequencies.items()))
+
+def euclidean_scale(frequencies):
+ """Scale a set of frequencies so they have a unit euclidean length
+
+ >>> sorted(euclidean_scale({1: 1, 2: 0}).items())
+ [(1, 1.0), (2, 0.0)]
+ >>> sorted(euclidean_scale({1: 1, 2: 1}).items()) # doctest: +ELLIPSIS
+ [(1, 0.7071067...), (2, 0.7071067...)]
+ >>> sorted(euclidean_scale({1: 1, 2: 1, 3: 1}).items()) # doctest: +ELLIPSIS
+ [(1, 0.577350...), (2, 0.577350...), (3, 0.577350...)]
+ >>> sorted(euclidean_scale({1: 1, 2: 2, 3: 1}).items()) # doctest: +ELLIPSIS
+ [(1, 0.408248...), (2, 0.81649658...), (3, 0.408248...)]
+ """
+ length = sum([f ** 2 for f in frequencies.values()]) ** 0.5
+ return collections.defaultdict(int, ((k, v / length)
+ for (k, v) in frequencies.items()))
+
+
+def scale(frequencies):
+ """Scale a set of frequencies so the largest is 1
+
+ >>> sorted(scale({1: 1, 2: 0}).items())
+ [(1, 1.0), (2, 0.0)]
+ >>> sorted(scale({1: 1, 2: 1}).items())
+ [(1, 1.0), (2, 1.0)]
+ >>> sorted(scale({1: 1, 2: 1, 3: 1}).items())
+ [(1, 1.0), (2, 1.0), (3, 1.0)]
+ >>> sorted(scale({1: 1, 2: 2, 3: 1}).items())
+ [(1, 0.5), (2, 1.0), (3, 0.5)]
+ """
+ largest = max(frequencies.values())
+ return collections.defaultdict(int, ((k, v / largest)
+ for (k, v) in frequencies.items()))
+
+
+def l2(frequencies1, frequencies2):
+ """Finds the distances between two frequency profiles, expressed as dictionaries.
+ Assumes every key in frequencies1 is also in frequencies2
+
+ >>> l2({'a':1, 'b':1, 'c':1}, {'a':1, 'b':1, 'c':1})
+ 0.0
+ >>> l2({'a':2, 'b':2, 'c':2}, {'a':1, 'b':1, 'c':1}) # doctest: +ELLIPSIS
+ 1.73205080...
+ >>> l2(normalise({'a':2, 'b':2, 'c':2}), normalise({'a':1, 'b':1, 'c':1}))
+ 0.0
+ >>> l2({'a':0, 'b':2, 'c':0}, {'a':1, 'b':1, 'c':1}) # doctest: +ELLIPSIS
+ 1.732050807...
+ >>> l2(normalise({'a':0, 'b':2, 'c':0}), \
+ normalise({'a':1, 'b':1, 'c':1})) # doctest: +ELLIPSIS
+ 0.81649658...
+ >>> l2({'a':0, 'b':1}, {'a':1, 'b':1})
+ 1.0
+ """
+ total = 0
+ for k in frequencies1.keys():
+ total += (frequencies1[k] - frequencies2[k]) ** 2
+ return total ** 0.5
+euclidean_distance = l2
+
+def l1(frequencies1, frequencies2):
+ """Finds the distances between two frequency profiles, expressed as
+ dictionaries. Assumes every key in frequencies1 is also in frequencies2
+
+ >>> l1({'a':1, 'b':1, 'c':1}, {'a':1, 'b':1, 'c':1})
+ 0
+ >>> l1({'a':2, 'b':2, 'c':2}, {'a':1, 'b':1, 'c':1})
+ 3
+ >>> l1(normalise({'a':2, 'b':2, 'c':2}), normalise({'a':1, 'b':1, 'c':1}))
+ 0.0
+ >>> l1({'a':0, 'b':2, 'c':0}, {'a':1, 'b':1, 'c':1})
+ 3
+ >>> l1({'a':0, 'b':1}, {'a':1, 'b':1})
+ 1
+ """
+ total = 0
+ for k in frequencies1.keys():
+ total += abs(frequencies1[k] - frequencies2[k])
+ return total
+
+def l3(frequencies1, frequencies2):
+ """Finds the distances between two frequency profiles, expressed as
+ dictionaries. Assumes every key in frequencies1 is also in frequencies2
+
+ >>> l3({'a':1, 'b':1, 'c':1}, {'a':1, 'b':1, 'c':1})
+ 0.0
+ >>> l3({'a':2, 'b':2, 'c':2}, {'a':1, 'b':1, 'c':1}) # doctest: +ELLIPSIS
+ 1.44224957...
+ >>> l3({'a':0, 'b':2, 'c':0}, {'a':1, 'b':1, 'c':1}) # doctest: +ELLIPSIS
+ 1.4422495703...
+ >>> l3(normalise({'a':0, 'b':2, 'c':0}), \
+ normalise({'a':1, 'b':1, 'c':1})) # doctest: +ELLIPSIS
+ 0.718144896...
+ >>> l3({'a':0, 'b':1}, {'a':1, 'b':1})
+ 1.0
+ >>> l3(normalise({'a':0, 'b':1}), normalise({'a':1, 'b':1})) # doctest: +ELLIPSIS
+ 0.6299605249...
+ """
+ total = 0
+ for k in frequencies1.keys():
+ total += abs(frequencies1[k] - frequencies2[k]) ** 3
+ return total ** (1/3)
+
+def geometric_mean(frequencies1, frequencies2):
+ """Finds the geometric mean of the absolute differences between two frequency profiles,
+ expressed as dictionaries.
+ Assumes every key in frequencies1 is also in frequencies2
+
+ >>> geometric_mean({'a':2, 'b':2, 'c':2}, {'a':1, 'b':1, 'c':1})
+ 1
+ >>> geometric_mean({'a':2, 'b':2, 'c':2}, {'a':1, 'b':1, 'c':1})
+ 1
+ >>> geometric_mean({'a':2, 'b':2, 'c':2}, {'a':1, 'b':5, 'c':1})
+ 3
+ >>> geometric_mean(normalise({'a':2, 'b':2, 'c':2}), \
+ normalise({'a':1, 'b':5, 'c':1})) # doctest: +ELLIPSIS
+ 0.01382140...
+ >>> geometric_mean(normalise({'a':2, 'b':2, 'c':2}), \
+ normalise({'a':1, 'b':1, 'c':1})) # doctest: +ELLIPSIS
+ 0.0
+ >>> geometric_mean(normalise({'a':2, 'b':2, 'c':2}), \
+ normalise({'a':1, 'b':1, 'c':0})) # doctest: +ELLIPSIS
+ 0.009259259...
+ """
+ total = 1
+ for k in frequencies1.keys():
+ total *= abs(frequencies1[k] - frequencies2[k])
+ return total
+
+def harmonic_mean(frequencies1, frequencies2):
+ """Finds the harmonic mean of the absolute differences between two frequency profiles,
+ expressed as dictionaries.
+ Assumes every key in frequencies1 is also in frequencies2
+
+ >>> harmonic_mean({'a':2, 'b':2, 'c':2}, {'a':1, 'b':1, 'c':1})
+ 1.0
+ >>> harmonic_mean({'a':2, 'b':2, 'c':2}, {'a':1, 'b':1, 'c':1})
+ 1.0
+ >>> harmonic_mean({'a':2, 'b':2, 'c':2}, {'a':1, 'b':5, 'c':1}) # doctest: +ELLIPSIS
+ 1.285714285...
+ >>> harmonic_mean(normalise({'a':2, 'b':2, 'c':2}), \
+ normalise({'a':1, 'b':5, 'c':1})) # doctest: +ELLIPSIS
+ 0.228571428571...
+ >>> harmonic_mean(normalise({'a':2, 'b':2, 'c':2}), \
+ normalise({'a':1, 'b':1, 'c':1})) # doctest: +ELLIPSIS
+ 0
+ >>> harmonic_mean(normalise({'a':2, 'b':2, 'c':2}), \
+ normalise({'a':1, 'b':1, 'c':0})) # doctest: +ELLIPSIS
+ 0.2
+ """
+ total = 0
+ for k in frequencies1.keys():
+ if abs(frequencies1[k] - frequencies2[k]) == 0:
+ return 0
+ total += 1 / abs(frequencies1[k] - frequencies2[k])
+ return len(frequencies1) / total
+
+
+def cosine_distance(frequencies1, frequencies2):
+ """Finds the distances between two frequency profiles, expressed as dictionaries.
+ Assumes every key in frequencies1 is also in frequencies2
+
+ >>> cosine_distance({'a':1, 'b':1, 'c':1}, {'a':1, 'b':1, 'c':1}) # doctest: +ELLIPSIS
+ -2.22044604...e-16
+ >>> cosine_distance({'a':2, 'b':2, 'c':2}, {'a':1, 'b':1, 'c':1}) # doctest: +ELLIPSIS
+ -2.22044604...e-16
+ >>> cosine_distance({'a':0, 'b':2, 'c':0}, {'a':1, 'b':1, 'c':1}) # doctest: +ELLIPSIS
+ 0.4226497308...
+ >>> cosine_distance({'a':0, 'b':1}, {'a':1, 'b':1}) # doctest: +ELLIPSIS
+ 0.29289321881...
+ """
+ numerator = 0
+ length1 = 0
+ length2 = 0
+ for k in frequencies1.keys():
+ numerator += frequencies1[k] * frequencies2[k]
+ length1 += frequencies1[k]**2
+ for k in frequencies2.keys():
+ length2 += frequencies2[k]
+ return 1 - (numerator / (length1 ** 0.5 * length2 ** 0.5))
+
+
+def index_of_coincidence(frequencies):
+ """Finds the (expected) index of coincidence given a set of frequencies
+ """
+ return sum([f ** 2 for f in frequencies.values()]) * len(frequencies.keys())
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod()
--- /dev/null
+# import re, string, random, glob, operator, heapq
+import string
+import collections
+from math import log10
+import itertools
+import sys
+from functools import lru_cache
+sys.setrecursionlimit(1000000)
+
+@lru_cache()
+def segment(text):
+ """Return a list of words that is the best segmentation of text.
+ """
+ if not text: return []
+ candidates = ([first]+segment(rest) for first,rest in splits(text))
+ return max(candidates, key=Pwords)
+
+def splits(text, L=20):
+ """Return a list of all possible (first, rest) pairs, len(first)<=L.
+ """
+ return [(text[:i+1], text[i+1:])
+ for i in range(min(len(text), L))]
+
+def Pwords(words):
+ """The Naive Bayes log probability of a sequence of words.
+ """
+ return sum(Pw[w.lower()] for w in words)
+
+class Pdist(dict):
+ """A probability distribution estimated from counts in datafile.
+ Values are stored and returned as log probabilities.
+ """
+ def __init__(self, data=[], estimate_of_missing=None):
+ data1, data2 = itertools.tee(data)
+ self.total = sum([int(d[1]) for d in data1])
+ for key, count in data2:
+ self[key] = log10(int(count) / self.total)
+ self.estimate_of_missing = estimate_of_missing or (lambda k, N: 1./N)
+ def __missing__(self, key):
+ return self.estimate_of_missing(key, self.total)
+
+def datafile(name, sep='\t'):
+ """Read key,value pairs from file.
+ """
+ with open(name, 'r') as f:
+ for line in f:
+ yield line.split(sep)
+
+def avoid_long_words(key, N):
+ """Estimate the probability of an unknown word.
+ """
+ return -log10((N * 10**(len(key) - 2)))
+
+Pw = Pdist(datafile('count_1w.txt'), avoid_long_words)
+
projects / cipher-tools.git / commitdiff