projects
/
cipher-tools.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Added a couple of other experiments
[cipher-tools.git]
/
support
/
segment.py
diff --git
a/support/segment.py
b/support/segment.py
index ba3ddd7405a91a40c025fcd34b5eadfa7f8d0b11..5bdb06cee8ffee28058f698879643c47218c4e7b 100644
(file)
--- a/
support/segment.py
+++ b/
support/segment.py
@@
-1,6
+1,7
@@
-import language_models
import sys
from functools import lru_cache
import sys
from functools import lru_cache
+from support.language_models import Pwords
+
sys.setrecursionlimit(1000000)
@lru_cache()
sys.setrecursionlimit(1000000)
@lru_cache()
@@
-9,7
+10,7
@@
def segment(text):
"""
if not text: return []
candidates = ([first]+segment(rest) for first,rest in splits(text))
"""
if not text: return []
candidates = ([first]+segment(rest) for first,rest in splits(text))
- return max(candidates, key=
language_models.
Pwords)
+ return max(candidates, key=Pwords)
def splits(text, L=20):
"""Return a list of all possible (first, rest) pairs, len(first)<=L.
def splits(text, L=20):
"""Return a list of all possible (first, rest) pairs, len(first)<=L.