From 6dae779d336388fdec43f684e9607c25423d1572 Mon Sep 17 00:00:00 2001 From: Neil Smith Date: Thu, 22 Oct 2020 15:49:17 +0100 Subject: [PATCH] Now using importlib --- .../support => raw_data_processing}/lettercount.py | 0 .../make-cracking-dictionary.py | 0 .../support => raw_data_processing}/shakespeare.txt | 0 .../sherlock-holmes.txt | 0 .../support => raw_data_processing}/war-and-peace.txt | 0 setup.py | 2 +- szyfrow/language_model_files/__init__.py | 0 .../{support => language_model_files}/count_1l.txt | 0 .../{support => language_model_files}/count_1w.txt | 0 .../{support => language_model_files}/count_2l.txt | 0 .../{support => language_model_files}/count_2w.txt | 0 .../{support => language_model_files}/count_3l.txt | 0 .../{support => language_model_files}/count_big.txt | 0 szyfrow/{support => language_model_files}/words.txt | 0 szyfrow/support/language_models.py | 11 +++++++++-- 15 files changed, 10 insertions(+), 3 deletions(-) rename {szyfrow/support => raw_data_processing}/lettercount.py (100%) rename {szyfrow/support => raw_data_processing}/make-cracking-dictionary.py (100%) rename {szyfrow/support => raw_data_processing}/shakespeare.txt (100%) rename {szyfrow/support => raw_data_processing}/sherlock-holmes.txt (100%) rename {szyfrow/support => raw_data_processing}/war-and-peace.txt (100%) create mode 100644 szyfrow/language_model_files/__init__.py rename szyfrow/{support => language_model_files}/count_1l.txt (100%) rename szyfrow/{support => language_model_files}/count_1w.txt (100%) rename szyfrow/{support => language_model_files}/count_2l.txt (100%) rename szyfrow/{support => language_model_files}/count_2w.txt (100%) rename szyfrow/{support => language_model_files}/count_3l.txt (100%) rename szyfrow/{support => language_model_files}/count_big.txt (100%) rename szyfrow/{support => language_model_files}/words.txt (100%) diff --git a/szyfrow/support/lettercount.py b/raw_data_processing/lettercount.py similarity index 100% rename from szyfrow/support/lettercount.py rename to raw_data_processing/lettercount.py diff --git a/szyfrow/support/make-cracking-dictionary.py b/raw_data_processing/make-cracking-dictionary.py similarity index 100% rename from szyfrow/support/make-cracking-dictionary.py rename to raw_data_processing/make-cracking-dictionary.py diff --git a/szyfrow/support/shakespeare.txt b/raw_data_processing/shakespeare.txt similarity index 100% rename from szyfrow/support/shakespeare.txt rename to raw_data_processing/shakespeare.txt diff --git a/szyfrow/support/sherlock-holmes.txt b/raw_data_processing/sherlock-holmes.txt similarity index 100% rename from szyfrow/support/sherlock-holmes.txt rename to raw_data_processing/sherlock-holmes.txt diff --git a/szyfrow/support/war-and-peace.txt b/raw_data_processing/war-and-peace.txt similarity index 100% rename from szyfrow/support/war-and-peace.txt rename to raw_data_processing/war-and-peace.txt diff --git a/setup.py b/setup.py index bb9b329..2df3ed0 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="szyfrow", - version="0.0.2", + version="0.0.3", author="Neil Smith", author_email="neil.szyfrow@njae.me.uk", description="Tools for using and breaking simple ciphers", diff --git a/szyfrow/language_model_files/__init__.py b/szyfrow/language_model_files/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/szyfrow/support/count_1l.txt b/szyfrow/language_model_files/count_1l.txt similarity index 100% rename from szyfrow/support/count_1l.txt rename to szyfrow/language_model_files/count_1l.txt diff --git a/szyfrow/support/count_1w.txt b/szyfrow/language_model_files/count_1w.txt similarity index 100% rename from szyfrow/support/count_1w.txt rename to szyfrow/language_model_files/count_1w.txt diff --git a/szyfrow/support/count_2l.txt b/szyfrow/language_model_files/count_2l.txt similarity index 100% rename from szyfrow/support/count_2l.txt rename to szyfrow/language_model_files/count_2l.txt diff --git a/szyfrow/support/count_2w.txt b/szyfrow/language_model_files/count_2w.txt similarity index 100% rename from szyfrow/support/count_2w.txt rename to szyfrow/language_model_files/count_2w.txt diff --git a/szyfrow/support/count_3l.txt b/szyfrow/language_model_files/count_3l.txt similarity index 100% rename from szyfrow/support/count_3l.txt rename to szyfrow/language_model_files/count_3l.txt diff --git a/szyfrow/support/count_big.txt b/szyfrow/language_model_files/count_big.txt similarity index 100% rename from szyfrow/support/count_big.txt rename to szyfrow/language_model_files/count_big.txt diff --git a/szyfrow/support/words.txt b/szyfrow/language_model_files/words.txt similarity index 100% rename from szyfrow/support/words.txt rename to szyfrow/language_model_files/words.txt diff --git a/szyfrow/support/language_models.py b/szyfrow/support/language_models.py index 271b8db..6898eb4 100644 --- a/szyfrow/support/language_models.py +++ b/szyfrow/support/language_models.py @@ -5,13 +5,20 @@ import itertools from math import log10 import os +import importlib.resources as pkg_resources + import szyfrow.support.norms from szyfrow.support.utilities import sanitise + +from szyfrow import language_model_files + + def datafile(name, sep='\t'): """Read key,value pairs from file. """ - with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), name), 'r') as f: + with pkg_resources.open_text(language_model_files, name) as f: + # with open(p name), 'r') as f: for line in f: splits = line.split(sep) yield [splits[0], int(splits[1])] @@ -25,7 +32,7 @@ normalised_english_bigram_counts = szyfrow.support.norms.normalise(english_bigra english_trigram_counts = collections.Counter(dict(datafile('count_3l.txt'))) normalised_english_trigram_counts = szyfrow.support.norms.normalise(english_trigram_counts) -with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'words.txt'), 'r') as f: +with pkg_resources.open_text(language_model_files, 'words.txt') as f: keywords = [line.rstrip() for line in f] -- 2.34.1