profiling/modules.md

   1 ---
   2 jupyter:
   3   jupytext:
   4     formats: ipynb,md
   5     text_representation:
   6       extension: .md
   7       format_name: markdown
   8       format_version: '1.3'
   9       jupytext_version: 1.11.1
  10   kernelspec:
  11     display_name: Python 3 (ipykernel)
  12     language: python
  13     name: python3
  14 ---
  15
  16 ```python
  17 import os, glob
  18 import collections
  19 import pandas as pd
  20 import numpy as np
  21
  22 import matplotlib as mpl
  23 import matplotlib.pyplot as plt
  24 %matplotlib inline
  25 ```
  26
  27 ```python
  28 with open('../advent-of-code22.cabal') as f:
  29     build_depends = [l for l in f.readlines() if 'build-depends' in l]
  30 build_depends
  31 ```
  32
  33 ```python
  34 cabal_file = open('../advent-of-code22.cabal').read()
  35 executables = cabal_file.split('executable')[2:]
  36 executables[:3]
  37 ```
  38
  39 ```python
  40 e = executables[1]
  41 e.strip().split('build-depends: ')[1].split(',')
  42 ```
  43
  44 ```python
  45 def extract(line):
  46     parts = line.strip().split('build-depends: ')
  47     name = parts[0].split()[0]
  48     if len(parts) > 1:
  49         depends = [p.strip() for p in parts[1].split('\n')[0].split(',') if 'base' not in p]
  50     else:
  51         depends = []
  52     return name, depends
  53 ```
  54
  55 ```python
  56 modules = {e: ms for e, ms in [extract(e) for e in executables] if e.endswith(tuple(str(i) for i in range(10)))}
  57 modules
  58 ```
  59
  60 ```python
  61 all_modules = set(m for p in modules for m in modules[p])
  62 modules_df = pd.DataFrame.from_dict({p: {m: m in modules[p] for m in sorted(all_modules)} for p in modules}, orient='index').sort_index()
  63 modules_df
  64 ```
  65
  66 ```python
  67 print(modules_df.sum().sort_values(ascending=False).to_markdown())
  68 ```
  69
  70 ```python tags=[]
  71 sorted_modules = modules_df.sum().sort_values(ascending=False).index.values
  72 sorted_modules
  73 ```
  74
  75 ```python tags=[]
  76 modules_sorted_cols = modules_df[sorted_modules]
  77 modules_sorted_cols
  78 ```
  79
  80 ```python
  81 modules_scatter = modules_df.stack().reset_index()
  82 modules_scatter.columns = ['program', 'module', 'present']
  83 modules_scatter = modules_scatter[modules_scatter.present]
  84 modules_scatter
  85 ```
  86
  87 ```python tags=[]
  88 modules_scatter.plot.scatter(x='program', y='module', s=80, rot=45, figsize=(10, 6))
  89 ```
  90
  91 ```python
  92 cmap = mpl.colors.ListedColormap(['white', 'blue'])
  93
  94 fig, ax = plt.subplots(figsize=(10, 10))
  95 ax.imshow(modules_df.to_numpy().T, cmap=cmap)
  96 plt.xticks(range(modules_df.index.size), labels=modules_df.index.values, rotation=90);
  97 plt.yticks(range(modules_df.columns.size), labels=modules_df.columns.values);
  98
  99 ax.xaxis.set_minor_locator(mpl.ticker.MultipleLocator(0.5))
 100 ax.yaxis.set_minor_locator(mpl.ticker.MultipleLocator(0.5))
 101 ax.grid(which='minor', axis='both', linestyle='-', color='silver', linewidth=1.5);
 102 plt.savefig('packages.png');
 103 ```
 104
 105 ```python
 106 cmap = mpl.colors.ListedColormap(['white', 'blue'])
 107
 108 fig, ax = plt.subplots(figsize=(10, 10))
 109 ax.imshow(modules_sorted_cols.to_numpy().T, cmap=cmap)
 110 plt.xticks(range(modules_sorted_cols.index.size), labels=modules_sorted_cols.index.values, rotation=90);
 111 plt.yticks(range(modules_sorted_cols.columns.size), labels=modules_sorted_cols.columns.values);
 112
 113 ax.xaxis.set_minor_locator(mpl.ticker.MultipleLocator(0.5))
 114 ax.yaxis.set_minor_locator(mpl.ticker.MultipleLocator(0.5))
 115 ax.grid(which='minor', axis='both', linestyle='-', color='silver', linewidth=1.5);
 116 plt.savefig('packages_sorted.png');
 117 ```
 118
 119 ```python
 120 mains = list(sorted(f for f in glob.glob('../advent*/Main.hs')))
 121 mains
 122 ```
 123
 124 ```python
 125 main_imports = {}
 126
 127 for m in mains:
 128     with open(m) as f:
 129         lines = f.readlines()
 130         import_lines = [l for l in lines if l.strip().startswith('import') if 'Debug.Trace' not in l]
 131         imports = []
 132         for i in import_lines:
 133             words = i.strip().split()
 134             if 'qualified' in i:
 135                 imports.append((words[2], True))
 136             else:
 137                 imports.append((words[1], False))
 138     main_imports[m.split('/')[1]] = imports
 139
 140 main_imports
 141 ```
 142
 143 ```python
 144 import_counts = collections.Counter(l for ls in main_imports.values() for l in ls)
 145 import_counts.most_common()
 146 ```
 147
 148 ```python
 149 main_imports_unqualified = {m: set(i[0] for i in main_imports[m]) for m in main_imports}
 150 main_imports_unqualified
 151 ```
 152
 153 ```python
 154 import_counts_unqualified = collections.Counter(l for ls in main_imports_unqualified.values() for l in ls)
 155 import_counts_unqualified.most_common()
 156 ```
 157
 158 ```python
 159 all_imports = set(m for p in main_imports_unqualified for m in main_imports_unqualified[p])
 160 imports_df = pd.DataFrame.from_dict(
 161     {p: {m: m in main_imports_unqualified[p]
 162          for m in sorted(all_imports)}
 163      for p in main_imports_unqualified},
 164     orient='index').sort_index()
 165 imports_df
 166 ```
 167
 168 ```python
 169 print(imports_df.sum().sort_values(ascending=False).to_markdown())
 170 ```
 171
 172 ```python
 173 imports_scatter = imports_df.stack().reset_index()
 174 imports_scatter.columns = ['program', 'module', 'present']
 175 imports_scatter = imports_scatter[imports_scatter.present]
 176 imports_scatter
 177 ```
 178
 179 ```python tags=[]
 180 imports_scatter.plot.scatter(x='program', y='module', s=80, rot=45, figsize=(10, 10))
 181 ```
 182
 183 ```python
 184 imports_df.columns.size
 185 ```
 186
 187 ```python tags=[]
 188 sorted_imports = imports_df.sum().sort_values(ascending=False).index.values
 189 sorted_imports
 190 ```
 191
 192 ```python tags=[]
 193 imports_sorted_cols = imports_df[sorted_imports]
 194 imports_sorted_cols
 195 ```
 196
 197 ```python
 198 cmap = mpl.colors.ListedColormap(['white', 'blue'])
 199
 200 fig, ax = plt.subplots(figsize=(10, 10))
 201 ax.imshow(imports_df.to_numpy().T, cmap=cmap)
 202 plt.xticks(range(imports_df.index.size), labels=imports_df.index.values, rotation=90);
 203 plt.yticks(range(imports_df.columns.size), labels=imports_df.columns.values);
 204
 205 ax.xaxis.set_minor_locator(mpl.ticker.MultipleLocator(0.5))
 206 ax.yaxis.set_minor_locator(mpl.ticker.MultipleLocator(0.5))
 207 ax.grid(which='minor', axis='both', linestyle='-', color='silver', linewidth=1.5);
 208 plt.savefig('imports.png');
 209 ```
 210
 211 ```python
 212 cmap = mpl.colors.ListedColormap(['white', 'blue'])
 213
 214 fig, ax = plt.subplots(figsize=(10, 10))
 215 ax.imshow(imports_sorted_cols.to_numpy().T, cmap=cmap)
 216 plt.xticks(range(imports_sorted_cols.index.size), labels=imports_sorted_cols.index.values, rotation=90);
 217 plt.yticks(range(imports_sorted_cols.columns.size), labels=imports_sorted_cols.columns.values);
 218
 219 ax.xaxis.set_minor_locator(mpl.ticker.MultipleLocator(0.5))
 220 ax.yaxis.set_minor_locator(mpl.ticker.MultipleLocator(0.5))
 221 ax.grid(which='minor', axis='both', linestyle='-', color='silver', linewidth=1.5);
 222 plt.savefig('imports_sorted.png');
 223 ```
 224
 225 ```python
 226 import matplotlib as mpl
 227 mpl.__version__
 228 ```
 229
 230 ```python
 231
 232 ```