many_word_lists/make-combined-list.ipynb

   1 {
   2  "cells": [
   3   {
   4    "cell_type": "code",
   5    "execution_count": 1,
   6    "metadata": {},
   7    "outputs": [],
   8    "source": [
   9     "import os,sys,inspect\n",
  10     "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
  11     "parentdir = os.path.dirname(currentdir)\n",
  12     "sys.path.insert(0,parentdir) \n",
  13     "\n",
  14     "from support.utilities import *"
  15    ]
  16   },
  17   {
  18    "cell_type": "code",
  19    "execution_count": 6,
  20    "metadata": {},
  21    "outputs": [
  22     {
  23      "data": {
  24       "text/plain": [
  25        "279585"
  26       ]
  27      },
  28      "execution_count": 6,
  29      "metadata": {},
  30      "output_type": "execute_result"
  31     }
  32    ],
  33    "source": [
  34     "many_words = set()\n",
  35     "\n",
  36     "for file in os.listdir():\n",
  37     "    if file.endswith(\".txt\") or file.endswith('list'):\n",
  38     "        these_words = set(sanitise(w) for w in open(file))\n",
  39     "        many_words.update(these_words)\n",
  40     "\n",
  41     "len(many_words)"
  42    ]
  43   },
  44   {
  45    "cell_type": "code",
  46    "execution_count": 5,
  47    "metadata": {},
  48    "outputs": [],
  49    "source": [
  50     "with open('combined.txt', 'w') as f:\n",
  51     "    for w in many_words:\n",
  52     "        f.write(w + '\\n')"
  53    ]
  54   },
  55   {
  56    "cell_type": "code",
  57    "execution_count": null,
  58    "metadata": {},
  59    "outputs": [],
  60    "source": []
  61   }
  62  ],
  63  "metadata": {
  64   "kernelspec": {
  65    "display_name": "Python 3",
  66    "language": "python",
  67    "name": "python3"
  68   },
  69   "language_info": {
  70    "codemirror_mode": {
  71     "name": "ipython",
  72     "version": 3
  73    },
  74    "file_extension": ".py",
  75    "mimetype": "text/x-python",
  76    "name": "python",
  77    "nbconvert_exporter": "python",
  78    "pygments_lexer": "ipython3",
  79    "version": "3.6.7"
  80   }
  81  },
  82  "nbformat": 4,
  83  "nbformat_minor": 2
  84 }