X-Git-Url: https://git.njae.me.uk/?a=blobdiff_plain;f=support%2Fsegment.py;h=5bdb06cee8ffee28058f698879643c47218c4e7b;hb=1350633e419f802975c773db8de86e3de49ea7fd;hp=ba3ddd7405a91a40c025fcd34b5eadfa7f8d0b11;hpb=df271527a6e3e4d10da9f9be3d1e9ebea9171493;p=cipher-tools.git diff --git a/support/segment.py b/support/segment.py index ba3ddd7..5bdb06c 100644 --- a/support/segment.py +++ b/support/segment.py @@ -1,6 +1,7 @@ -import language_models import sys from functools import lru_cache +from support.language_models import Pwords + sys.setrecursionlimit(1000000) @lru_cache() @@ -9,7 +10,7 @@ def segment(text): """ if not text: return [] candidates = ([first]+segment(rest) for first,rest in splits(text)) - return max(candidates, key=language_models.Pwords) + return max(candidates, key=Pwords) def splits(text, L=20): """Return a list of all possible (first, rest) pairs, len(first)<=L.