projects / covid19.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Updated for imported data format
[covid19.git] / covid-old.ipynb
1 {
2 "cells": [
3 {
4 "cell_type": "markdown",
5 "metadata": {
6 "Collapsed": "false"
7 },
8 "source": [
9 "Data from [European Centre for Disease Prevention and Control](https://www.ecdc.europa.eu/en/publications-data/download-todays-data-geographic-distribution-covid-19-cases-worldwide)"
10 ]
11 },
12 {
13 "cell_type": "code",
14 "execution_count": 1,
15 "metadata": {
16 "Collapsed": "false"
17 },
18 "outputs": [],
19 "source": [
20 "import itertools\n",
21 "import collections\n",
22 "import json\n",
23 "import pandas as pd\n",
24 "import numpy as np\n",
25 "from scipy.stats import gmean\n",
26 "import datetime\n",
27 "\n",
28 "import matplotlib as mpl\n",
29 "import matplotlib.pyplot as plt\n",
30 "%matplotlib inline"
31 ]
32 },
33 {
34 "cell_type": "code",
35 "execution_count": 2,
36 "metadata": {
37 "Collapsed": "false"
38 },
39 "outputs": [],
40 "source": [
41 "DEATH_COUNT_THRESHOLD = 10\n",
42 "COUNTRIES_CORE = 'IT DE UK ES IE FR BE'.split()\n",
43 "COUNTRIES_NORDIC = 'SE NO DK FI UK'.split()\n",
44 "COUNTRIES_FRIENDS = 'IT UK ES BE SI MX'.split()\n",
45 "# COUNTRIES_FRIENDS = 'IT UK ES BE SI PT'.split()\n",
46 "\n",
47 "COUNTRIES_AMERICAS = ['AG', 'AR', 'AW', 'BS', 'BB', 'BZ', 'BM', 'BO', 'BR', 'VG', 'KY', # excluding Canada and USA\n",
48 " 'CL', 'CO', 'CR', 'CU', 'CW', 'DM', 'DO', 'EC', 'SV', 'GL', 'GD', 'GT',\n",
49 " 'GY', 'HT', 'HN', 'JM', 'MX', 'MS', 'NI', 'PA', 'PY', 'PE', 'PR', 'KN',\n",
50 " 'LC', 'VC', 'SX', 'SR', 'TT', 'TC', 'VI', 'UY', 'VE']\n",
51 "COUNTRIES_OF_INTEREST = list(set(COUNTRIES_CORE + COUNTRIES_FRIENDS))\n",
52 "COUNTRIES_ALL = list(set(COUNTRIES_CORE + COUNTRIES_FRIENDS + COUNTRIES_NORDIC + COUNTRIES_AMERICAS))"
53 ]
54 },
55 {
56 "cell_type": "code",
57 "execution_count": 4843,
58 "metadata": {
59 "Collapsed": "false"
60 },
61 "outputs": [
62 {
63 "name": "stdout",
64 "output_type": "stream",
65 "text": [
66 " % Total % Received % Xferd Average Speed Time Time Time Current\n",
67 " Dload Upload Total Spent Left Speed\n",
68 "100 553k 100 553k 0 0 564k 0 --:--:-- --:--:-- --:--:-- 564k\n"
69 ]
70 }
71 ],
72 "source": [
73 "!curl https://opendata.ecdc.europa.eu/covid19/casedistribution/csv/ > covid.csv"
74 ]
75 },
76 {
77 "cell_type": "code",
78 "execution_count": 4844,
79 "metadata": {
80 "Collapsed": "false"
81 },
82 "outputs": [],
83 "source": [
84 "# First col is a date, treat geoId of NA as 'Namibia', not \"NA\" value\n",
85 "raw_data = pd.read_csv('covid.csv', \n",
86 " parse_dates=[0], dayfirst=True,\n",
87 " keep_default_na=False, na_values = [''],\n",
88 "# dtype = {'day': np.int64, \n",
89 "# 'month': np.int64, \n",
90 "# 'year': np.int64, \n",
91 "# 'cases': np.int64, \n",
92 "# 'deaths': np.int64, \n",
93 "# 'countriesAndTerritories': str, \n",
94 "# 'geoId': str, \n",
95 "# 'countryterritoryCode': str, \n",
96 "# 'popData2019': np.int64, \n",
97 "# 'continentExp': str, \n",
98 "# }\n",
99 " )"
100 ]
101 },
102 {
103 "cell_type": "code",
104 "execution_count": 4845,
105 "metadata": {
106 "Collapsed": "false"
107 },
108 "outputs": [
109 {
110 "data": {
111 "text/plain": [
112 "89150"
113 ]
114 },
115 "execution_count": 4845,
116 "metadata": {},
117 "output_type": "execute_result"
118 }
119 ],
120 "source": [
121 "raw_data.size"
122 ]
123 },
124 {
125 "cell_type": "code",
126 "execution_count": 4846,
127 "metadata": {
128 "Collapsed": "false"
129 },
130 "outputs": [],
131 "source": [
132 "raw_data.fillna(0, inplace=True)"
133 ]
134 },
135 {
136 "cell_type": "code",
137 "execution_count": 4847,
138 "metadata": {
139 "Collapsed": "false"
140 },
141 "outputs": [
142 {
143 "data": {
144 "text/html": [
145 "<div>\n",
146 "<style scoped>\n",
147 " .dataframe tbody tr th:only-of-type {\n",
148 " vertical-align: middle;\n",
149 " }\n",
150 "\n",
151 " .dataframe tbody tr th {\n",
152 " vertical-align: top;\n",
153 " }\n",
154 "\n",
155 " .dataframe thead th {\n",
156 " text-align: right;\n",
157 " }\n",
158 "</style>\n",
159 "<table border=\"1\" class=\"dataframe\">\n",
160 " <thead>\n",
161 " <tr style=\"text-align: right;\">\n",
162 " <th></th>\n",
163 " <th>dateRep</th>\n",
164 " <th>year_week</th>\n",
165 " <th>cases_weekly</th>\n",
166 " <th>deaths_weekly</th>\n",
167 " <th>countriesAndTerritories</th>\n",
168 " <th>geoId</th>\n",
169 " <th>countryterritoryCode</th>\n",
170 " <th>popData2019</th>\n",
171 " <th>continentExp</th>\n",
172 " <th>notification_rate_per_100000_population_14-days</th>\n",
173 " </tr>\n",
174 " </thead>\n",
175 " <tbody>\n",
176 " <tr>\n",
177 " <td>0</td>\n",
178 " <td>2020-12-14</td>\n",
179 " <td>2020-50</td>\n",
180 " <td>1757</td>\n",
181 " <td>71</td>\n",
182 " <td>Afghanistan</td>\n",
183 " <td>AF</td>\n",
184 " <td>AFG</td>\n",
185 " <td>38041757.0</td>\n",
186 " <td>Asia</td>\n",
187 " <td>9.01</td>\n",
188 " </tr>\n",
189 " <tr>\n",
190 " <td>1</td>\n",
191 " <td>2020-12-07</td>\n",
192 " <td>2020-49</td>\n",
193 " <td>1672</td>\n",
194 " <td>137</td>\n",
195 " <td>Afghanistan</td>\n",
196 " <td>AF</td>\n",
197 " <td>AFG</td>\n",
198 " <td>38041757.0</td>\n",
199 " <td>Asia</td>\n",
200 " <td>7.22</td>\n",
201 " </tr>\n",
202 " <tr>\n",
203 " <td>2</td>\n",
204 " <td>2020-11-30</td>\n",
205 " <td>2020-48</td>\n",
206 " <td>1073</td>\n",
207 " <td>68</td>\n",
208 " <td>Afghanistan</td>\n",
209 " <td>AF</td>\n",
210 " <td>AFG</td>\n",
211 " <td>38041757.0</td>\n",
212 " <td>Asia</td>\n",
213 " <td>6.42</td>\n",
214 " </tr>\n",
215 " <tr>\n",
216 " <td>3</td>\n",
217 " <td>2020-11-23</td>\n",
218 " <td>2020-47</td>\n",
219 " <td>1368</td>\n",
220 " <td>69</td>\n",
221 " <td>Afghanistan</td>\n",
222 " <td>AF</td>\n",
223 " <td>AFG</td>\n",
224 " <td>38041757.0</td>\n",
225 " <td>Asia</td>\n",
226 " <td>6.66</td>\n",
227 " </tr>\n",
228 " <tr>\n",
229 " <td>4</td>\n",
230 " <td>2020-11-16</td>\n",
231 " <td>2020-46</td>\n",
232 " <td>1164</td>\n",
233 " <td>61</td>\n",
234 " <td>Afghanistan</td>\n",
235 " <td>AF</td>\n",
236 " <td>AFG</td>\n",
237 " <td>38041757.0</td>\n",
238 " <td>Asia</td>\n",
239 " <td>4.65</td>\n",
240 " </tr>\n",
241 " </tbody>\n",
242 "</table>\n",
243 "</div>"
244 ],
245 "text/plain": [
246 " dateRep year_week cases_weekly deaths_weekly countriesAndTerritories \\\n",
247 "0 2020-12-14 2020-50 1757 71 Afghanistan \n",
248 "1 2020-12-07 2020-49 1672 137 Afghanistan \n",
249 "2 2020-11-30 2020-48 1073 68 Afghanistan \n",
250 "3 2020-11-23 2020-47 1368 69 Afghanistan \n",
251 "4 2020-11-16 2020-46 1164 61 Afghanistan \n",
252 "\n",
253 " geoId countryterritoryCode popData2019 continentExp \\\n",
254 "0 AF AFG 38041757.0 Asia \n",
255 "1 AF AFG 38041757.0 Asia \n",
256 "2 AF AFG 38041757.0 Asia \n",
257 "3 AF AFG 38041757.0 Asia \n",
258 "4 AF AFG 38041757.0 Asia \n",
259 "\n",
260 " notification_rate_per_100000_population_14-days \n",
261 "0 9.01 \n",
262 "1 7.22 \n",
263 "2 6.42 \n",
264 "3 6.66 \n",
265 "4 4.65 "
266 ]
267 },
268 "execution_count": 4847,
269 "metadata": {},
270 "output_type": "execute_result"
271 }
272 ],
273 "source": [
274 "raw_data.head()"
275 ]
276 },
277 {
278 "cell_type": "code",
279 "execution_count": 4848,
280 "metadata": {
281 "Collapsed": "false"
282 },
283 "outputs": [
284 {
285 "data": {
286 "text/plain": [
287 "dateRep datetime64[ns]\n",
288 "year_week object\n",
289 "cases_weekly int64\n",
290 "deaths_weekly int64\n",
291 "countriesAndTerritories object\n",
292 "geoId object\n",
293 "countryterritoryCode object\n",
294 "popData2019 float64\n",
295 "continentExp object\n",
296 "notification_rate_per_100000_population_14-days float64\n",
297 "dtype: object"
298 ]
299 },
300 "execution_count": 4848,
301 "metadata": {},
302 "output_type": "execute_result"
303 }
304 ],
305 "source": [
306 "raw_data.dtypes"
307 ]
308 },
309 {
310 "cell_type": "code",
311 "execution_count": 4849,
312 "metadata": {
313 "Collapsed": "false"
314 },
315 "outputs": [
316 {
317 "ename": "KeyError",
318 "evalue": "'Only a column name can be used for the key in a dtype mappings argument.'",
319 "output_type": "error",
320 "traceback": [
321 "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
322 "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
323 "\u001b[0;32m<ipython-input-4849-4ccdef75f4f0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;34m'countryterritoryCode'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;34m'popData2019'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint64\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m 'continentExp': str })\n\u001b[0m",
324 "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36mastype\u001b[0;34m(self, dtype, copy, errors, **kwargs)\u001b[0m\n\u001b[1;32m 5855\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcol_name\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5856\u001b[0m raise KeyError(\n\u001b[0;32m-> 5857\u001b[0;31m \u001b[0;34m\"Only a column name can be used for the \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5858\u001b[0m \u001b[0;34m\"key in a dtype mappings argument.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5859\u001b[0m )\n",
325 "\u001b[0;31mKeyError\u001b[0m: 'Only a column name can be used for the key in a dtype mappings argument.'"
326 ]
327 }
328 ],
329 "source": [
330 "# raw_data = raw_data.astype({'dateRep': np.datetime64, \n",
331 "# 'day': np.int64, \n",
332 "# 'month': np.int64, \n",
333 "# 'year': np.int64, \n",
334 "# 'cases': np.int64, \n",
335 "# 'deaths': np.int64, \n",
336 "# 'countriesAndTerritories': str, \n",
337 "# 'geoId': str, \n",
338 "# 'countryterritoryCode': str, \n",
339 "# 'popData2019': np.int64, \n",
340 "# 'continentExp': str })\n",
341 "raw_data = raw_data.astype({'dateRep': np.datetime64, \n",
342 " 'day': np.int64, \n",
343 " 'month': np.int64, \n",
344 " 'year': np.int64, \n",
345 " 'cases': np.int64, \n",
346 " 'deaths': np.int64, \n",
347 " 'countriesAndTerritories': str, \n",
348 " 'geoId': str, \n",
349 " 'countryterritoryCode': str, \n",
350 " 'popData2019': np.int64, \n",
351 " 'continentExp': str })"
352 ]
353 },
354 {
355 "cell_type": "code",
356 "execution_count": null,
357 "metadata": {
358 "Collapsed": "false"
359 },
360 "outputs": [],
361 "source": [
362 "raw_data.dtypes"
363 ]
364 },
365 {
366 "cell_type": "code",
367 "execution_count": null,
368 "metadata": {
369 "Collapsed": "false"
370 },
371 "outputs": [],
372 "source": [
373 "raw_data[((raw_data.geoId == 'UK') & (raw_data.dateRep >= '2020-07-10'))]"
374 ]
375 },
376 {
377 "cell_type": "code",
378 "execution_count": null,
379 "metadata": {
380 "Collapsed": "false"
381 },
382 "outputs": [],
383 "source": [
384 "# raw_data = raw_data[~ ((raw_data.geoId == 'ES') & (raw_data.dateRep >= '2020-05-22'))]"
385 ]
386 },
387 {
388 "cell_type": "code",
389 "execution_count": null,
390 "metadata": {
391 "Collapsed": "false"
392 },
393 "outputs": [],
394 "source": [
395 "base_data = raw_data.set_index(['geoId', 'dateRep'])\n",
396 "base_data.sort_index(inplace=True)\n",
397 "base_data"
398 ]
399 },
400 {
401 "cell_type": "code",
402 "execution_count": null,
403 "metadata": {
404 "Collapsed": "false"
405 },
406 "outputs": [],
407 "source": [
408 "base_data.loc['ES'].loc['2020-05-10':]"
409 ]
410 },
411 {
412 "cell_type": "code",
413 "execution_count": null,
414 "metadata": {
415 "Collapsed": "false"
416 },
417 "outputs": [],
418 "source": [
419 "countries = raw_data[['geoId', 'countriesAndTerritories', 'popData2019', 'continentExp']]\n",
420 "countries = countries[countries['popData2019'] != '']\n",
421 "countries = countries.drop_duplicates()\n",
422 "countries.set_index('geoId', inplace=True)\n",
423 "countries = countries.astype({'popData2019': 'int64'})\n",
424 "countries.head()"
425 ]
426 },
427 {
428 "cell_type": "code",
429 "execution_count": null,
430 "metadata": {
431 "Collapsed": "false"
432 },
433 "outputs": [],
434 "source": [
435 "countries.shape"
436 ]
437 },
438 {
439 "cell_type": "code",
440 "execution_count": null,
441 "metadata": {
442 "Collapsed": "false"
443 },
444 "outputs": [],
445 "source": [
446 "countries[countries.countriesAndTerritories == 'Finland']"
447 ]
448 },
449 {
450 "cell_type": "code",
451 "execution_count": null,
452 "metadata": {
453 "Collapsed": "false"
454 },
455 "outputs": [],
456 "source": [
457 "countries.loc[COUNTRIES_OF_INTEREST]"
458 ]
459 },
460 {
461 "cell_type": "code",
462 "execution_count": null,
463 "metadata": {
464 "Collapsed": "false"
465 },
466 "outputs": [],
467 "source": [
468 "countries[countries.continentExp == 'America'].index"
469 ]
470 },
471 {
472 "cell_type": "code",
473 "execution_count": null,
474 "metadata": {
475 "Collapsed": "false"
476 },
477 "outputs": [],
478 "source": [
479 "data_by_date = base_data[['cases', 'deaths']]\n",
480 "data_by_date.head()"
481 ]
482 },
483 {
484 "cell_type": "code",
485 "execution_count": null,
486 "metadata": {
487 "Collapsed": "false"
488 },
489 "outputs": [],
490 "source": [
491 "data_by_date.loc['UK']"
492 ]
493 },
494 {
495 "cell_type": "code",
496 "execution_count": null,
497 "metadata": {
498 "Collapsed": "false"
499 },
500 "outputs": [],
501 "source": [
502 "# data_by_date.deaths.drop_duplicates().sort_values().to_csv('dth.csv', header=True)"
503 ]
504 },
505 {
506 "cell_type": "code",
507 "execution_count": null,
508 "metadata": {
509 "Collapsed": "false"
510 },
511 "outputs": [],
512 "source": [
513 "data_by_date.groupby(level=0).cumsum()"
514 ]
515 },
516 {
517 "cell_type": "code",
518 "execution_count": null,
519 "metadata": {
520 "Collapsed": "false"
521 },
522 "outputs": [],
523 "source": [
524 "data_by_date = data_by_date.merge(\n",
525 " data_by_date.groupby(level=0).cumsum(), \n",
526 " suffixes=('', '_culm'), \n",
527 " left_index=True, right_index=True)\n",
528 "data_by_date"
529 ]
530 },
531 {
532 "cell_type": "code",
533 "execution_count": null,
534 "metadata": {
535 "Collapsed": "false"
536 },
537 "outputs": [],
538 "source": [
539 "data_by_date = data_by_date.merge(\n",
540 " data_by_date[['cases', 'deaths']].groupby(level=0).diff(), \n",
541 " suffixes=('', '_diff'), \n",
542 " left_index=True, right_index=True)\n",
543 "data_by_date"
544 ]
545 },
546 {
547 "cell_type": "code",
548 "execution_count": null,
549 "metadata": {
550 "Collapsed": "false"
551 },
552 "outputs": [],
553 "source": [
554 "data_by_date.loc['UK', '2020-04-17']"
555 ]
556 },
557 {
558 "cell_type": "code",
559 "execution_count": null,
560 "metadata": {
561 "Collapsed": "false"
562 },
563 "outputs": [],
564 "source": [
565 "data_by_date.loc['UK']"
566 ]
567 },
568 {
569 "cell_type": "code",
570 "execution_count": null,
571 "metadata": {
572 "Collapsed": "false"
573 },
574 "outputs": [],
575 "source": [
576 "# data_by_date[data_by_date.deaths_culm > DEATH_COUNT_THRESHOLD]"
577 ]
578 },
579 {
580 "cell_type": "code",
581 "execution_count": null,
582 "metadata": {
583 "Collapsed": "false"
584 },
585 "outputs": [],
586 "source": [
587 "# days_since_threshold = data_by_date[data_by_date.deaths_culm > DEATH_COUNT_THRESHOLD].groupby(level=0).cumcount()\n",
588 "# days_since_threshold.rename('since_threshold', inplace=True)"
589 ]
590 },
591 {
592 "cell_type": "code",
593 "execution_count": null,
594 "metadata": {
595 "Collapsed": "false"
596 },
597 "outputs": [],
598 "source": [
599 "dbd = data_by_date[data_by_date.deaths_culm > DEATH_COUNT_THRESHOLD].reset_index(level=1)\n",
600 "dbd['since_threshold'] = dbd.dateRep\n",
601 "dbd.set_index('dateRep', append=True, inplace=True)\n",
602 "dbd.sort_index(inplace=True)\n",
603 "days_since_threshold = dbd.groupby(level=0).diff().since_threshold.dt.days.fillna(0).astype(int).groupby(level=0).cumsum()\n",
604 "# days_since_threshold.groupby(level=0).cumsum()\n",
605 "\n",
606 "# days_since_threshold = dbd.rename('since_threshold')\n",
607 "days_since_threshold"
608 ]
609 },
610 {
611 "cell_type": "code",
612 "execution_count": null,
613 "metadata": {
614 "Collapsed": "false"
615 },
616 "outputs": [],
617 "source": [
618 "# days_since_threshold = (data_by_date[data_by_date.deaths_culm > DEATH_COUNT_THRESHOLD]\n",
619 "# .reset_index(level=1).groupby(level=0)\n",
620 "# .diff().dateRep.dt.days\n",
621 "# .groupby(level=0).cumcount()\n",
622 "# )\n",
623 "# days_since_threshold.rename('since_threshold', inplace=True)\n",
624 "# days_since_threshold"
625 ]
626 },
627 {
628 "cell_type": "code",
629 "execution_count": null,
630 "metadata": {
631 "Collapsed": "false"
632 },
633 "outputs": [],
634 "source": [
635 "data_since_threshold = data_by_date.merge(days_since_threshold, \n",
636 " left_index=True, right_index=True)\n",
637 "data_since_threshold"
638 ]
639 },
640 {
641 "cell_type": "code",
642 "execution_count": null,
643 "metadata": {
644 "Collapsed": "false"
645 },
646 "outputs": [],
647 "source": [
648 "data_since_threshold = data_since_threshold.set_index('since_threshold', append=True\n",
649 " ).reorder_levels(['since_threshold', 'geoId', 'dateRep']\n",
650 " ).reset_index('dateRep')\n",
651 "data_since_threshold.sort_index(inplace=True)\n",
652 "data_since_threshold"
653 ]
654 },
655 {
656 "cell_type": "code",
657 "execution_count": null,
658 "metadata": {
659 "Collapsed": "false"
660 },
661 "outputs": [],
662 "source": [
663 "data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), :]"
664 ]
665 },
666 {
667 "cell_type": "code",
668 "execution_count": null,
669 "metadata": {
670 "Collapsed": "false"
671 },
672 "outputs": [],
673 "source": [
674 "data_since_threshold.loc[(slice(None), ['ES']), :].tail(8)"
675 ]
676 },
677 {
678 "cell_type": "code",
679 "execution_count": null,
680 "metadata": {
681 "Collapsed": "false"
682 },
683 "outputs": [],
684 "source": [
685 "data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), ['deaths_culm']].unstack().plot(logy=True)"
686 ]
687 },
688 {
689 "cell_type": "code",
690 "execution_count": null,
691 "metadata": {
692 "Collapsed": "false"
693 },
694 "outputs": [],
695 "source": [
696 "# deaths = data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT', 'IE']), ['deaths_culm']].unstack().xs('deaths_culm', axis=1, drop_level=True)"
697 ]
698 },
699 {
700 "cell_type": "code",
701 "execution_count": null,
702 "metadata": {
703 "Collapsed": "false"
704 },
705 "outputs": [],
706 "source": [
707 "deaths = data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['deaths_culm']].unstack().sort_index().xs('deaths_culm', axis=1, drop_level=True)\n",
708 "deaths_by_date = data_by_date.loc[COUNTRIES_ALL, ['deaths_culm']].unstack().sort_index().xs('deaths_culm', axis=1, drop_level=True).T"
709 ]
710 },
711 {
712 "cell_type": "code",
713 "execution_count": null,
714 "metadata": {
715 "Collapsed": "false"
716 },
717 "outputs": [],
718 "source": [
719 "cases = data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['cases_culm']].unstack().sort_index().xs('cases_culm', axis=1, drop_level=True)\n",
720 "cases_by_date = data_by_date.loc[ COUNTRIES_ALL, ['cases_culm']].unstack().sort_index().xs('cases_culm', axis=1, drop_level=True).T"
721 ]
722 },
723 {
724 "cell_type": "code",
725 "execution_count": null,
726 "metadata": {
727 "Collapsed": "false"
728 },
729 "outputs": [],
730 "source": [
731 "COUNTRIES_AMERICAS_DEAD = list(set(deaths.columns) & set(COUNTRIES_AMERICAS))"
732 ]
733 },
734 {
735 "cell_type": "code",
736 "execution_count": null,
737 "metadata": {
738 "Collapsed": "false"
739 },
740 "outputs": [],
741 "source": [
742 "data_since_threshold.reset_index().merge(countries, on='geoId').set_index(['since_threshold', 'geoId'])"
743 ]
744 },
745 {
746 "cell_type": "code",
747 "execution_count": null,
748 "metadata": {
749 "Collapsed": "false"
750 },
751 "outputs": [],
752 "source": [
753 "data_since_threshold.reset_index().merge(countries, on='geoId').set_index(['since_threshold', 'geoId']).sort_index(inplace=True)"
754 ]
755 },
756 {
757 "cell_type": "code",
758 "execution_count": null,
759 "metadata": {
760 "Collapsed": "false"
761 },
762 "outputs": [],
763 "source": [
764 "data_since_threshold_per_capita = data_since_threshold.reset_index().merge(countries, on='geoId').set_index(['since_threshold', 'geoId'])\n",
765 "data_since_threshold_per_capita['cases_culm_pc'] = data_since_threshold_per_capita.cases_culm / data_since_threshold_per_capita.popData2019\n",
766 "data_since_threshold_per_capita['deaths_culm_pc'] = data_since_threshold_per_capita.deaths_culm / data_since_threshold_per_capita.popData2019\n",
767 "data_since_threshold_per_capita"
768 ]
769 },
770 {
771 "cell_type": "code",
772 "execution_count": null,
773 "metadata": {
774 "Collapsed": "false"
775 },
776 "outputs": [],
777 "source": [
778 "deaths_pc = data_since_threshold_per_capita.loc[(slice(None), ['UK', 'DE', 'IT', 'IE']), ['deaths_culm_pc']].unstack().sort_index().xs('deaths_culm_pc', axis=1, drop_level=True)"
779 ]
780 },
781 {
782 "cell_type": "code",
783 "execution_count": null,
784 "metadata": {
785 "Collapsed": "false"
786 },
787 "outputs": [],
788 "source": [
789 "deaths_pc.index"
790 ]
791 },
792 {
793 "cell_type": "code",
794 "execution_count": null,
795 "metadata": {
796 "Collapsed": "false"
797 },
798 "outputs": [],
799 "source": [
800 "deaths_pc = data_since_threshold_per_capita.loc[(slice(None), COUNTRIES_ALL), ['deaths_culm_pc']].unstack().xs('deaths_culm_pc', axis=1, drop_level=True)"
801 ]
802 },
803 {
804 "cell_type": "code",
805 "execution_count": null,
806 "metadata": {
807 "Collapsed": "false"
808 },
809 "outputs": [],
810 "source": [
811 "deaths[COUNTRIES_CORE].plot()"
812 ]
813 },
814 {
815 "cell_type": "code",
816 "execution_count": null,
817 "metadata": {
818 "Collapsed": "false"
819 },
820 "outputs": [],
821 "source": [
822 "deaths[COUNTRIES_FRIENDS].plot()"
823 ]
824 },
825 {
826 "cell_type": "code",
827 "execution_count": null,
828 "metadata": {
829 "Collapsed": "false"
830 },
831 "outputs": [],
832 "source": [
833 "ax = deaths[COUNTRIES_FRIENDS].plot(figsize=(10, 6), title=\"Total deaths, linear\")\n",
834 "ax.set_xlabel(f\"Days since {DEATH_COUNT_THRESHOLD} deaths\")\n",
835 "for c in COUNTRIES_FRIENDS:\n",
836 " lvi = deaths[c].last_valid_index()\n",
837 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = f\"{c}: {deaths[c][lvi]:.0f}\")\n",
838 "# plt.savefig('covid_deaths_total_linear.png') "
839 ]
840 },
841 {
842 "cell_type": "code",
843 "execution_count": null,
844 "metadata": {
845 "Collapsed": "false"
846 },
847 "outputs": [],
848 "source": [
849 "ax = deaths[COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Total deaths, linear\")\n",
850 "ax.set_xlabel(f\"Days since {DEATH_COUNT_THRESHOLD} deaths\")\n",
851 "for c in COUNTRIES_CORE:\n",
852 " lvi = deaths[c].last_valid_index()\n",
853 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = f\"{c}: {deaths[c][lvi]:.0f}\")\n",
854 "# plt.savefig('covid_deaths_total_linear.png') "
855 ]
856 },
857 {
858 "cell_type": "code",
859 "execution_count": null,
860 "metadata": {
861 "Collapsed": "false"
862 },
863 "outputs": [],
864 "source": [
865 "ax = deaths_by_date.loc['2020-03-15':, COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Total deaths, linear\")\n",
866 "# data_by_date.loc[COUNTRIES_CORE]\n",
867 "# deaths_by_date = data_by_date.loc[COUNTRIES_ALL, ['deaths_culm']].unstack().sort_index().xs('deaths_culm', axis=1, drop_level=True)\n",
868 "ax.set_xlabel(f\"Date\")\n",
869 "for c in COUNTRIES_CORE:\n",
870 " lvi = deaths_by_date[c].last_valid_index()\n",
871 " ax.text(x = lvi + pd.Timedelta(days=1), y = deaths_by_date[c][lvi], s = f\"{c}: {deaths_by_date[c][lvi]:.0f}\")\n",
872 "plt.savefig('covid_deaths_total_linear.png') "
873 ]
874 },
875 {
876 "cell_type": "code",
877 "execution_count": null,
878 "metadata": {
879 "Collapsed": "false"
880 },
881 "outputs": [],
882 "source": [
883 "deaths_prime = deaths[COUNTRIES_CORE].copy()\n",
884 "deaths_prime.loc[73:, 'ES'] = np.NaN\n",
885 "# deaths_prime['ES'][70:]"
886 ]
887 },
888 {
889 "cell_type": "code",
890 "execution_count": null,
891 "metadata": {
892 "Collapsed": "false"
893 },
894 "outputs": [],
895 "source": [
896 "ax = deaths_prime[COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Total deaths, linear\")\n",
897 "for c in COUNTRIES_CORE:\n",
898 " lvi = deaths_prime[c].last_valid_index()\n",
899 " ax.text(x = lvi + 1, y = deaths_prime[c][lvi], s = f\"{c}: {deaths_prime[c][lvi]:.0f}\")\n",
900 "# plt.savefig('covid_deaths_total_linear.png') "
901 ]
902 },
903 {
904 "cell_type": "code",
905 "execution_count": null,
906 "metadata": {
907 "Collapsed": "false"
908 },
909 "outputs": [],
910 "source": [
911 "ax = cases[COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Total cases, linear\")\n",
912 "for c in COUNTRIES_CORE:\n",
913 " lvi = cases[c].last_valid_index()\n",
914 " ax.text(x = lvi + 1, y = cases[c][lvi], s = c)\n",
915 "plt.savefig('covid_cases_total_linear.png') "
916 ]
917 },
918 {
919 "cell_type": "code",
920 "execution_count": null,
921 "metadata": {
922 "Collapsed": "false"
923 },
924 "outputs": [],
925 "source": [
926 "ax = deaths[COUNTRIES_AMERICAS_DEAD].plot(figsize=(10, 6), title=\"Total deaths, linear\")\n",
927 "for c in COUNTRIES_AMERICAS_DEAD:\n",
928 " lvi = deaths[c].last_valid_index()\n",
929 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)\n",
930 "# plt.savefig('covid_deaths_total_linear.png') "
931 ]
932 },
933 {
934 "cell_type": "code",
935 "execution_count": null,
936 "metadata": {
937 "Collapsed": "false"
938 },
939 "outputs": [],
940 "source": [
941 "ax = deaths[COUNTRIES_CORE + ['BR', 'MX']].plot(figsize=(10, 6), title=\"Total deaths, linear\")\n",
942 "for c in COUNTRIES_CORE + ['BR', 'MX']:\n",
943 " lvi = deaths[c].last_valid_index()\n",
944 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)\n",
945 "# plt.savefig('covid_deaths_total_linear.png') "
946 ]
947 },
948 {
949 "cell_type": "code",
950 "execution_count": null,
951 "metadata": {
952 "Collapsed": "false"
953 },
954 "outputs": [],
955 "source": [
956 "ax = deaths[COUNTRIES_NORDIC].plot(figsize=(10, 6), title=\"Total deaths, linear\")\n",
957 "for c in COUNTRIES_NORDIC:\n",
958 " lvi = deaths[c].last_valid_index()\n",
959 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)\n",
960 "# plt.savefig('covid_deaths_total_linear.png') "
961 ]
962 },
963 {
964 "cell_type": "code",
965 "execution_count": null,
966 "metadata": {
967 "Collapsed": "false"
968 },
969 "outputs": [],
970 "source": [
971 "ax = deaths[COUNTRIES_OF_INTEREST].plot(figsize=(10, 6), title=\"Total deaths, linear\")\n",
972 "for c in COUNTRIES_OF_INTEREST:\n",
973 " lvi = deaths[c].last_valid_index()\n",
974 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)\n",
975 "plt.savefig('covid_deaths_total_linear_of_interest.png') "
976 ]
977 },
978 {
979 "cell_type": "code",
980 "execution_count": null,
981 "metadata": {
982 "Collapsed": "false"
983 },
984 "outputs": [],
985 "source": [
986 "ax = deaths[COUNTRIES_CORE].plot(logy=True, figsize=(10, 6), title=\"Total deaths, log\")\n",
987 "for c in COUNTRIES_CORE:\n",
988 " lvi = deaths[c].last_valid_index()\n",
989 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)\n",
990 "\n",
991 "plt.savefig('covid_deaths_total_log.png')"
992 ]
993 },
994 {
995 "cell_type": "code",
996 "execution_count": null,
997 "metadata": {
998 "Collapsed": "false"
999 },
1000 "outputs": [],
1001 "source": [
1002 "ylim = (5*10**3, 5*10**4)\n",
1003 "ax = deaths[COUNTRIES_CORE].plot(logy=True, figsize=(10, 6), ylim=ylim, title=\"Total deaths, log\")\n",
1004 "for c in COUNTRIES_CORE:\n",
1005 " lvi = deaths[c].last_valid_index()\n",
1006 " if ylim[0] < deaths[c][lvi] < ylim[1]:\n",
1007 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)\n",
1008 "\n",
1009 "# plt.savefig('covid_deaths_total_log.png')"
1010 ]
1011 },
1012 {
1013 "cell_type": "code",
1014 "execution_count": null,
1015 "metadata": {
1016 "Collapsed": "false"
1017 },
1018 "outputs": [],
1019 "source": [
1020 "ax = deaths[COUNTRIES_FRIENDS].plot(logy=True, figsize=(10, 6), title=\"Total deaths, log\")\n",
1021 "for c in COUNTRIES_FRIENDS:\n",
1022 " lvi = deaths[c].last_valid_index()\n",
1023 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)\n",
1024 "\n",
1025 "# plt.savefig('covid_deaths_total_log.png')"
1026 ]
1027 },
1028 {
1029 "cell_type": "code",
1030 "execution_count": null,
1031 "metadata": {
1032 "Collapsed": "false"
1033 },
1034 "outputs": [],
1035 "source": [
1036 "ax = deaths[COUNTRIES_NORDIC].plot(logy=True, figsize=(10, 6), title=\"Total deaths, log\")\n",
1037 "for c in COUNTRIES_NORDIC:\n",
1038 " lvi = deaths[c].last_valid_index()\n",
1039 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)\n",
1040 "\n",
1041 "# plt.savefig('covid_deaths_total_log.png')"
1042 ]
1043 },
1044 {
1045 "cell_type": "code",
1046 "execution_count": null,
1047 "metadata": {
1048 "Collapsed": "false"
1049 },
1050 "outputs": [],
1051 "source": [
1052 "ax = deaths[COUNTRIES_OF_INTEREST].plot(logy=True, figsize=(10, 6), title=\"Total deaths, log\")\n",
1053 "for c in COUNTRIES_OF_INTEREST:\n",
1054 " lvi = deaths[c].last_valid_index()\n",
1055 " ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)\n",
1056 "\n",
1057 "plt.savefig('covid_deaths_total_log.png')"
1058 ]
1059 },
1060 {
1061 "cell_type": "code",
1062 "execution_count": null,
1063 "metadata": {
1064 "Collapsed": "false"
1065 },
1066 "outputs": [],
1067 "source": [
1068 "ax = deaths_pc.plot(figsize=(10, 6), title=\"Deaths per capita, linear\")\n",
1069 "for c in deaths_pc.columns:\n",
1070 " lvi = deaths_pc[c].last_valid_index()\n",
1071 " ax.text(x = lvi + 1, y = deaths_pc[c][lvi], s = c)\n",
1072 "plt.savefig('covid_deaths_per_capita_linear.png')"
1073 ]
1074 },
1075 {
1076 "cell_type": "code",
1077 "execution_count": null,
1078 "metadata": {
1079 "Collapsed": "false"
1080 },
1081 "outputs": [],
1082 "source": [
1083 "ax = deaths_pc.plot(logy=True, figsize=(10, 6), title=\"Deaths per capita, log\")\n",
1084 "for c in deaths_pc.columns:\n",
1085 " lvi = deaths_pc[c].last_valid_index()\n",
1086 " ax.text(x = lvi + 1, y = deaths_pc[c][lvi], s = c)"
1087 ]
1088 },
1089 {
1090 "cell_type": "code",
1091 "execution_count": null,
1092 "metadata": {
1093 "Collapsed": "false"
1094 },
1095 "outputs": [],
1096 "source": [
1097 "deaths_pc[['UK', 'IE']].plot( figsize=(10, 6), title=\"Deaths per capita, linear\")"
1098 ]
1099 },
1100 {
1101 "cell_type": "code",
1102 "execution_count": null,
1103 "metadata": {
1104 "Collapsed": "false"
1105 },
1106 "outputs": [],
1107 "source": [
1108 "deaths_pc[['UK', 'IE']].plot(logy=True, figsize=(10, 6), title=\"Deaths per capita, log\")"
1109 ]
1110 },
1111 {
1112 "cell_type": "code",
1113 "execution_count": null,
1114 "metadata": {
1115 "Collapsed": "false"
1116 },
1117 "outputs": [],
1118 "source": [
1119 "deaths[['UK', 'ES', 'IT']].plot(logy=True, figsize=(10, 6), title=\"Deaths, log\")\n",
1120 "plt.savefig('covid_deaths_selected_log.png')"
1121 ]
1122 },
1123 {
1124 "cell_type": "code",
1125 "execution_count": null,
1126 "metadata": {
1127 "Collapsed": "false"
1128 },
1129 "outputs": [],
1130 "source": [
1131 "deaths[['UK', 'ES', 'IT', 'MX']].plot(logy=True, figsize=(10, 6), title=\"Deaths, log\")"
1132 ]
1133 },
1134 {
1135 "cell_type": "code",
1136 "execution_count": null,
1137 "metadata": {
1138 "Collapsed": "false"
1139 },
1140 "outputs": [],
1141 "source": [
1142 "data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), :]"
1143 ]
1144 },
1145 {
1146 "cell_type": "code",
1147 "execution_count": null,
1148 "metadata": {
1149 "Collapsed": "false"
1150 },
1151 "outputs": [],
1152 "source": [
1153 "data_since_threshold['deaths_m4'] = data_since_threshold.groupby(level=1)['deaths'].transform(lambda x: x.rolling(4, 1).mean())\n",
1154 "data_since_threshold['deaths_m7'] = data_since_threshold.groupby(level=1)['deaths'].transform(lambda x: x.rolling(7, 1).mean())\n",
1155 "data_since_threshold['cases_m7'] = data_since_threshold.groupby(level=1)['cases'].transform(lambda x: x.rolling(7, 1).mean())\n",
1156 "# data_since_threshold['deaths_diff_m4'] = data_since_threshold.groupby(level=1)['deaths_diff'].transform(lambda x: x.rolling(4, 1).mean())\n",
1157 "# data_since_threshold['deaths_diff_m7'] = data_since_threshold.groupby(level=1)['deaths_diff'].transform(lambda x: x.rolling(7, 1).mean())\n",
1158 "data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), :]"
1159 ]
1160 },
1161 {
1162 "cell_type": "code",
1163 "execution_count": null,
1164 "metadata": {
1165 "Collapsed": "false"
1166 },
1167 "outputs": [],
1168 "source": [
1169 "deaths_m4 = (data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['deaths_m4']]\n",
1170 " .unstack().sort_index().xs('deaths_m4', axis=1, drop_level=True))"
1171 ]
1172 },
1173 {
1174 "cell_type": "code",
1175 "execution_count": null,
1176 "metadata": {
1177 "Collapsed": "false"
1178 },
1179 "outputs": [],
1180 "source": [
1181 "deaths_m7 = (data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['deaths_m7']]\n",
1182 " .unstack().sort_index().xs('deaths_m7', axis=1, drop_level=True))"
1183 ]
1184 },
1185 {
1186 "cell_type": "code",
1187 "execution_count": null,
1188 "metadata": {
1189 "Collapsed": "false"
1190 },
1191 "outputs": [],
1192 "source": [
1193 "cases_m7 = (data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['cases_m7']]\n",
1194 " .unstack().sort_index().xs('cases_m7', axis=1, drop_level=True))"
1195 ]
1196 },
1197 {
1198 "cell_type": "code",
1199 "execution_count": null,
1200 "metadata": {
1201 "Collapsed": "false"
1202 },
1203 "outputs": [],
1204 "source": [
1205 "data_by_date['cases_m7'] = data_by_date.groupby(level=0)['cases'].transform(lambda x: x.rolling(7, 1).mean())\n",
1206 "data_by_date['deaths_m7'] = data_by_date.groupby(level=0)['deaths'].transform(lambda x: x.rolling(7, 1).mean())\n",
1207 "data_by_date"
1208 ]
1209 },
1210 {
1211 "cell_type": "code",
1212 "execution_count": null,
1213 "metadata": {
1214 "Collapsed": "false"
1215 },
1216 "outputs": [],
1217 "source": [
1218 "data_by_date.loc[('UK', '2020-07-15'):'UK', 'cases'].plot()"
1219 ]
1220 },
1221 {
1222 "cell_type": "code",
1223 "execution_count": null,
1224 "metadata": {
1225 "Collapsed": "false"
1226 },
1227 "outputs": [],
1228 "source": [
1229 "cases_by_date_m7 = data_by_date.loc[COUNTRIES_ALL, 'cases_m7'].unstack(level=0).sort_index()\n",
1230 "cases_by_date_m7[COUNTRIES_CORE].plot()"
1231 ]
1232 },
1233 {
1234 "cell_type": "code",
1235 "execution_count": null,
1236 "metadata": {
1237 "Collapsed": "false"
1238 },
1239 "outputs": [],
1240 "source": [
1241 "deaths_by_date_m7 = data_by_date.loc[COUNTRIES_ALL, 'deaths_m7'].unstack(level=0).sort_index()\n",
1242 "deaths_by_date_m7[COUNTRIES_CORE].plot()"
1243 ]
1244 },
1245 {
1246 "cell_type": "code",
1247 "execution_count": null,
1248 "metadata": {
1249 "Collapsed": "false"
1250 },
1251 "outputs": [],
1252 "source": [
1253 "ax = deaths_m4.plot(figsize=(10, 6), title=\"Deaths per day, 4 day moving average\")\n",
1254 "for c in deaths_m4.columns:\n",
1255 " lvi = deaths_m4[c].last_valid_index()\n",
1256 " ax.text(x = lvi + 1, y = deaths_m4[c][lvi], s = c)\n",
1257 "plt.savefig('covid_deaths_per_day.png') "
1258 ]
1259 },
1260 {
1261 "cell_type": "code",
1262 "execution_count": null,
1263 "metadata": {
1264 "Collapsed": "false"
1265 },
1266 "outputs": [],
1267 "source": [
1268 "ax = deaths_m4[COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Deaths per day, 4 day moving average\")\n",
1269 "for c in COUNTRIES_CORE:\n",
1270 " lvi = deaths_m4[c].last_valid_index()\n",
1271 " ax.text(x = lvi + 1, y = deaths_m4[c][lvi], s = c)\n",
1272 "plt.savefig('covid_deaths_per_day-core.png') "
1273 ]
1274 },
1275 {
1276 "cell_type": "code",
1277 "execution_count": null,
1278 "metadata": {
1279 "Collapsed": "false"
1280 },
1281 "outputs": [],
1282 "source": [
1283 "ax = deaths_m4[COUNTRIES_FRIENDS].plot(figsize=(10, 6), title=\"Deaths per day, 4 day moving average\")\n",
1284 "for c in COUNTRIES_FRIENDS:\n",
1285 " lvi = deaths_m4[c].last_valid_index()\n",
1286 " ax.text(x = lvi + 1, y = deaths_m4[c][lvi], s = c)\n",
1287 "# plt.savefig('covid_deaths_per_day-friends.png') "
1288 ]
1289 },
1290 {
1291 "cell_type": "code",
1292 "execution_count": null,
1293 "metadata": {
1294 "Collapsed": "false"
1295 },
1296 "outputs": [],
1297 "source": [
1298 "C7s = 'ES FR IT UK'.split()\n",
1299 "ax = deaths_m7[C7s].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\")\n",
1300 "for c in C7s:\n",
1301 " lvi = deaths_m7[c].last_valid_index()\n",
1302 " ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = c)\n",
1303 "# plt.savefig('covid_deaths_per_day-friends.png') "
1304 ]
1305 },
1306 {
1307 "cell_type": "code",
1308 "execution_count": null,
1309 "metadata": {
1310 "Collapsed": "false"
1311 },
1312 "outputs": [],
1313 "source": [
1314 "ax = deaths_m7[COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\")\n",
1315 "ax.set_xlabel(f\"Days since {DEATH_COUNT_THRESHOLD} deaths\")\n",
1316 "for c in COUNTRIES_CORE:\n",
1317 " lvi = deaths_m7[c].last_valid_index()\n",
1318 " ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = c)\n",
1319 "# plt.axhline(0, color='0.7')\n",
1320 "plt.savefig('covid_deaths_per_day_7.png') "
1321 ]
1322 },
1323 {
1324 "cell_type": "code",
1325 "execution_count": null,
1326 "metadata": {
1327 "Collapsed": "false"
1328 },
1329 "outputs": [],
1330 "source": [
1331 "ax = deaths_m7[COUNTRIES_FRIENDS].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\")\n",
1332 "ax.set_xlabel(f\"Days since {DEATH_COUNT_THRESHOLD} deaths\")\n",
1333 "for c in COUNTRIES_FRIENDS:\n",
1334 " lvi = deaths_m7[c].last_valid_index()\n",
1335 " ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = c)\n",
1336 "# plt.axhline(0, color='0.7')\n",
1337 "plt.savefig('covid_deaths_per_day-friends.png') "
1338 ]
1339 },
1340 {
1341 "cell_type": "code",
1342 "execution_count": null,
1343 "metadata": {
1344 "Collapsed": "false"
1345 },
1346 "outputs": [],
1347 "source": [
1348 "deaths_m7_prime = deaths_m7[COUNTRIES_CORE].copy()\n",
1349 "deaths_m7_prime.loc[73:, 'ES'] = np.NaN\n",
1350 "deaths_m7_prime['ES'][70:]"
1351 ]
1352 },
1353 {
1354 "cell_type": "code",
1355 "execution_count": null,
1356 "metadata": {
1357 "Collapsed": "false"
1358 },
1359 "outputs": [],
1360 "source": [
1361 "ax = deaths_m7_prime[COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\")\n",
1362 "for c in COUNTRIES_CORE:\n",
1363 " lvi = deaths_m7_prime[c].last_valid_index()\n",
1364 " ax.text(x = lvi + 1, y = deaths_m7_prime[c][lvi], s = c)\n",
1365 "plt.savefig('covid_deaths_per_day_7.png') # see below for where this is written, with the projection"
1366 ]
1367 },
1368 {
1369 "cell_type": "code",
1370 "execution_count": null,
1371 "metadata": {
1372 "Collapsed": "false"
1373 },
1374 "outputs": [],
1375 "source": [
1376 "ax = deaths_by_date_m7.loc['2020-03-01':, COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\")\n",
1377 "ax.set_xlabel('Date')\n",
1378 "for c in COUNTRIES_CORE:\n",
1379 " lvi = deaths_by_date_m7[c].last_valid_index()\n",
1380 " ax.text(x = lvi + pd.Timedelta(days=1), y = deaths_by_date_m7[c][lvi], s = c)\n",
1381 "plt.savefig('covid_deaths_per_day_7.png') "
1382 ]
1383 },
1384 {
1385 "cell_type": "code",
1386 "execution_count": null,
1387 "metadata": {
1388 "Collapsed": "false"
1389 },
1390 "outputs": [],
1391 "source": [
1392 "ax = deaths_m7[COUNTRIES_FRIENDS].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\")\n",
1393 "for c in COUNTRIES_FRIENDS:\n",
1394 " lvi = deaths_m7[c].last_valid_index()\n",
1395 " ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = c)\n",
1396 "plt.savefig('covid_deaths_per_day_friends_7.png') "
1397 ]
1398 },
1399 {
1400 "cell_type": "code",
1401 "execution_count": null,
1402 "metadata": {
1403 "Collapsed": "false"
1404 },
1405 "outputs": [],
1406 "source": [
1407 "ax = deaths_m7[COUNTRIES_CORE + ['BR', 'MX']].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\")\n",
1408 "for c in COUNTRIES_CORE + ['BR', 'MX']:\n",
1409 " lvi = deaths_m7[c].last_valid_index()\n",
1410 " ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = c)\n",
1411 "# plt.savefig('covid_deaths_per_day_7.png') "
1412 ]
1413 },
1414 {
1415 "cell_type": "code",
1416 "execution_count": null,
1417 "metadata": {
1418 "Collapsed": "false"
1419 },
1420 "outputs": [],
1421 "source": [
1422 "ax = deaths_by_date_m7.iloc[-30:][COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\")#, ylim=(-10, 100))\n",
1423 "ax.set_xlabel(\"Date\")\n",
1424 "\n",
1425 "text_x_pos = deaths_by_date_m7.last_valid_index() + pd.Timedelta(days=1)\n",
1426 "\n",
1427 "for c in COUNTRIES_CORE:\n",
1428 " lvi = deaths_by_date_m7[c].last_valid_index()\n",
1429 "# if c != 'ES':\n",
1430 " ax.text(x = text_x_pos, y = deaths_by_date_m7[c][lvi], s = f\"{c}: {deaths_by_date_m7[c][lvi]:.0f}\")\n",
1431 "plt.savefig('deaths_by_date_last_30_days.png') "
1432 ]
1433 },
1434 {
1435 "cell_type": "code",
1436 "execution_count": null,
1437 "metadata": {
1438 "Collapsed": "false"
1439 },
1440 "outputs": [],
1441 "source": [
1442 "ax = deaths_by_date_m7.iloc[-30:][COUNTRIES_FRIENDS].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\")#, ylim=(-10, 100))\n",
1443 "ax.set_xlabel(\"Date\")\n",
1444 "\n",
1445 "text_x_pos = deaths_by_date_m7.last_valid_index() + pd.Timedelta(days=1)\n",
1446 "\n",
1447 "for c in COUNTRIES_FRIENDS:\n",
1448 " lvi = deaths_by_date_m7[c].last_valid_index()\n",
1449 "# if c != 'ES':\n",
1450 " ax.text(x = text_x_pos, y = deaths_by_date_m7[c][lvi], s = f\"{c}: {deaths_by_date_m7[c][lvi]:.0f}\")\n",
1451 "plt.savefig('deaths_by_date_last_30_days_friends.png') "
1452 ]
1453 },
1454 {
1455 "cell_type": "code",
1456 "execution_count": null,
1457 "metadata": {
1458 "Collapsed": "false"
1459 },
1460 "outputs": [],
1461 "source": [
1462 "ax = cases_m7[COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Cases per day, 7 day moving average\")\n",
1463 "for c in COUNTRIES_CORE:\n",
1464 " lvi = cases_m7[c].last_valid_index()\n",
1465 " ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = c)\n",
1466 "plt.savefig('covid_cases_per_day-core.png') "
1467 ]
1468 },
1469 {
1470 "cell_type": "code",
1471 "execution_count": null,
1472 "metadata": {
1473 "Collapsed": "false"
1474 },
1475 "outputs": [],
1476 "source": [
1477 "ax = cases_m7[COUNTRIES_FRIENDS].plot(figsize=(10, 6), title=\"Cases per day, 7 day moving average\")\n",
1478 "for c in COUNTRIES_FRIENDS:\n",
1479 " lvi = cases_m7[c].last_valid_index()\n",
1480 " ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = c)\n",
1481 "plt.savefig('covid_cases_per_day-core.png') "
1482 ]
1483 },
1484 {
1485 "cell_type": "code",
1486 "execution_count": null,
1487 "metadata": {
1488 "Collapsed": "false"
1489 },
1490 "outputs": [],
1491 "source": [
1492 "ax = cases_by_date_m7.iloc[-30:][COUNTRIES_FRIENDS].plot(figsize=(10, 6), title=\"Cases per day, 7 day moving average\")#, ylim=(-10, 100))\n",
1493 "ax.set_xlabel(\"Date\")\n",
1494 "\n",
1495 "text_x_pos = cases_by_date_m7.last_valid_index() + pd.Timedelta(days=1)\n",
1496 "\n",
1497 "for c in COUNTRIES_FRIENDS:\n",
1498 " lvi = cases_by_date_m7[c].last_valid_index()\n",
1499 "# if c != 'ES':\n",
1500 " ax.text(x = text_x_pos, y = cases_by_date_m7[c][lvi], s = f\"{c}: {cases_by_date_m7[c][lvi]:.0f}\")\n",
1501 "plt.savefig('cases_by_date_last_30_days_friends.png') "
1502 ]
1503 },
1504 {
1505 "cell_type": "code",
1506 "execution_count": null,
1507 "metadata": {
1508 "Collapsed": "false"
1509 },
1510 "outputs": [],
1511 "source": [
1512 "def gmean_scale(items):\n",
1513 " return gmean(items) / items[-1]"
1514 ]
1515 },
1516 {
1517 "cell_type": "code",
1518 "execution_count": null,
1519 "metadata": {
1520 "Collapsed": "false"
1521 },
1522 "outputs": [],
1523 "source": [
1524 "def doubling_time(df):\n",
1525 " return np.log(2) / np.log((df.deaths_culm + df.deaths_g4) / df.deaths_culm)\n",
1526 "\n",
1527 "def doubling_time_7(df):\n",
1528 " return np.log(2) / np.log((df.deaths_culm + df.deaths_g7) / df.deaths_culm)"
1529 ]
1530 },
1531 {
1532 "cell_type": "code",
1533 "execution_count": null,
1534 "metadata": {
1535 "Collapsed": "false"
1536 },
1537 "outputs": [],
1538 "source": [
1539 "# data_since_threshold['deaths_g4'] = data_since_threshold.groupby(level=1)['deaths'].transform(lambda x: x.rolling(4, 1).apply(gmean_scale, raw=True))\n",
1540 "# data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), :]"
1541 ]
1542 },
1543 {
1544 "cell_type": "code",
1545 "execution_count": null,
1546 "metadata": {
1547 "Collapsed": "false"
1548 },
1549 "outputs": [],
1550 "source": [
1551 "data_since_threshold['deaths_g4'] = data_since_threshold.groupby(level=1)['deaths'].transform(lambda x: x.rolling(4, 1).apply(gmean, raw=True))\n",
1552 "data_since_threshold['deaths_g7'] = data_since_threshold.groupby(level=1)['deaths'].transform(lambda x: x.rolling(7, 1).apply(gmean, raw=True))\n",
1553 "data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), :]"
1554 ]
1555 },
1556 {
1557 "cell_type": "code",
1558 "execution_count": null,
1559 "metadata": {
1560 "Collapsed": "false"
1561 },
1562 "outputs": [],
1563 "source": [
1564 "data_since_threshold['doubling_time'] = data_since_threshold.groupby(level=1).apply(doubling_time).reset_index(level=0, drop=True).sort_index()\n",
1565 "data_since_threshold['doubling_time_7'] = data_since_threshold.groupby(level=1).apply(doubling_time_7).reset_index(level=0, drop=True).sort_index()\n",
1566 "# data_since_threshold.loc[(slice(None), 'UK'), :]"
1567 ]
1568 },
1569 {
1570 "cell_type": "code",
1571 "execution_count": null,
1572 "metadata": {
1573 "Collapsed": "false"
1574 },
1575 "outputs": [],
1576 "source": [
1577 "data_by_date['deaths_g4'] = data_by_date.groupby(level=0)['deaths'].transform(lambda x: x.rolling(4, 1).apply(gmean, raw=True))\n",
1578 "data_by_date['deaths_g7'] = data_by_date.groupby(level=0)['deaths'].transform(lambda x: x.rolling(7, 1).apply(gmean, raw=True))\n",
1579 "data_by_date['doubling_time'] = data_by_date.groupby(level=0).apply(doubling_time).reset_index(level=0, drop=True).sort_index()\n",
1580 "data_by_date['doubling_time_7'] = data_by_date.groupby(level=0).apply(doubling_time_7).reset_index(level=0, drop=True).sort_index()\n",
1581 "data_by_date.loc['UK']"
1582 ]
1583 },
1584 {
1585 "cell_type": "code",
1586 "execution_count": null,
1587 "metadata": {
1588 "Collapsed": "false"
1589 },
1590 "outputs": [],
1591 "source": [
1592 "doubling_times = (data_since_threshold.loc[(slice(None), COUNTRIES_OF_INTEREST), ['doubling_time']]\n",
1593 " .unstack().sort_index().xs('doubling_time', axis=1, drop_level=True))\n",
1594 "doubling_times.replace([np.inf, -np.inf], np.nan, inplace=True)"
1595 ]
1596 },
1597 {
1598 "cell_type": "code",
1599 "execution_count": null,
1600 "metadata": {
1601 "Collapsed": "false"
1602 },
1603 "outputs": [],
1604 "source": [
1605 "doubling_times_7 = (data_since_threshold.loc[(slice(None), COUNTRIES_OF_INTEREST), ['doubling_time_7']]\n",
1606 " .unstack().sort_index().xs('doubling_time_7', axis=1, drop_level=True))\n",
1607 "doubling_times_7.replace([np.inf, -np.inf], np.nan, inplace=True)"
1608 ]
1609 },
1610 {
1611 "cell_type": "code",
1612 "execution_count": null,
1613 "metadata": {
1614 "Collapsed": "false"
1615 },
1616 "outputs": [],
1617 "source": [
1618 "ax = doubling_times.plot(figsize=(10, 6), title=\"Doubling times, 4 day average\")\n",
1619 "for c in doubling_times.columns:\n",
1620 " lvi = doubling_times[c].last_valid_index()\n",
1621 " ax.text(x = lvi + 1, y = doubling_times[c][lvi], s = c)\n",
1622 "# plt.savefig('covid_deaths_per_day.png') "
1623 ]
1624 },
1625 {
1626 "cell_type": "code",
1627 "execution_count": null,
1628 "metadata": {
1629 "Collapsed": "false"
1630 },
1631 "outputs": [],
1632 "source": [
1633 "ax = doubling_times_7[COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Doubling times, 7 day average\")\n",
1634 "ax.legend(loc=\"upper left\")\n",
1635 "for c in COUNTRIES_CORE:\n",
1636 " lvi = doubling_times_7[c].last_valid_index()\n",
1637 " ax.text(x = lvi + 1, y = doubling_times_7[c][lvi], s = c)\n",
1638 "plt.savefig('covid_doubling_times_7.png') "
1639 ]
1640 },
1641 {
1642 "cell_type": "code",
1643 "execution_count": null,
1644 "metadata": {
1645 "Collapsed": "false"
1646 },
1647 "outputs": [],
1648 "source": [
1649 "ax = doubling_times[COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Doubling times, 4 day average\")\n",
1650 "for c in COUNTRIES_CORE:\n",
1651 " lvi = doubling_times[c].last_valid_index()\n",
1652 " ax.text(x = lvi + 1, y = doubling_times[c][lvi], s = c)\n",
1653 "plt.savefig('covid_doubling_times.png') "
1654 ]
1655 },
1656 {
1657 "cell_type": "code",
1658 "execution_count": null,
1659 "metadata": {
1660 "Collapsed": "false"
1661 },
1662 "outputs": [],
1663 "source": [
1664 "ax = doubling_times[COUNTRIES_FRIENDS].plot(figsize=(10, 6), title=\"Doubling times\")\n",
1665 "for c in COUNTRIES_FRIENDS:\n",
1666 " lvi = doubling_times[c].last_valid_index()\n",
1667 " ax.text(x = lvi + 1, y = doubling_times[c][lvi], s = c)\n",
1668 "plt.savefig('covid_doubling_times_friends.png')"
1669 ]
1670 },
1671 {
1672 "cell_type": "code",
1673 "execution_count": null,
1674 "metadata": {
1675 "Collapsed": "false"
1676 },
1677 "outputs": [],
1678 "source": [
1679 "ax = doubling_times[C7s].plot(figsize=(10, 6), title=\"Doubling times\")\n",
1680 "for c in C7s:\n",
1681 " lvi = doubling_times[c].last_valid_index()\n",
1682 " ax.text(x = lvi + 1, y = doubling_times[c][lvi], s = c)\n",
1683 "# plt.savefig('covid_doubling_times_friends.png')"
1684 ]
1685 },
1686 {
1687 "cell_type": "code",
1688 "execution_count": null,
1689 "metadata": {
1690 "Collapsed": "false"
1691 },
1692 "outputs": [],
1693 "source": [
1694 "# deaths_diff_m4 = (data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['deaths_diff_m4']]\n",
1695 "# .unstack().sort_index().xs('deaths_diff_m4', axis=1, drop_level=True))"
1696 ]
1697 },
1698 {
1699 "cell_type": "code",
1700 "execution_count": null,
1701 "metadata": {
1702 "Collapsed": "false"
1703 },
1704 "outputs": [],
1705 "source": [
1706 "# deaths_diff_m7 = (data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['deaths_diff_m7']]\n",
1707 "# .unstack().sort_index().xs('deaths_diff_m7', axis=1, drop_level=True))"
1708 ]
1709 },
1710 {
1711 "cell_type": "code",
1712 "execution_count": null,
1713 "metadata": {
1714 "Collapsed": "false"
1715 },
1716 "outputs": [],
1717 "source": [
1718 "# deaths_diff_m7"
1719 ]
1720 },
1721 {
1722 "cell_type": "code",
1723 "execution_count": null,
1724 "metadata": {
1725 "Collapsed": "false"
1726 },
1727 "outputs": [],
1728 "source": [
1729 "# data_since_threshold.replace([np.inf, -np.inf], np.nan).groupby(level=1).last().loc[COUNTRIES_ALL]#, [doubling_time]]"
1730 ]
1731 },
1732 {
1733 "cell_type": "code",
1734 "execution_count": null,
1735 "metadata": {
1736 "Collapsed": "false"
1737 },
1738 "outputs": [],
1739 "source": [
1740 "dstl = data_since_threshold.replace([np.inf, -np.inf], np.nan).groupby(level=1).last()\n",
1741 "dstl.loc[dstl.index.intersection(COUNTRIES_ALL)]"
1742 ]
1743 },
1744 {
1745 "cell_type": "code",
1746 "execution_count": null,
1747 "metadata": {
1748 "Collapsed": "false"
1749 },
1750 "outputs": [],
1751 "source": [
1752 "# data_since_threshold.replace([np.inf, -np.inf], np.nan).groupby(level=1).last().loc[['UK', 'DE', 'IT']]#, [doubling_time]]\n",
1753 "dstl.loc[['UK', 'DE', 'IT', 'FR', 'ES']]"
1754 ]
1755 },
1756 {
1757 "cell_type": "code",
1758 "execution_count": null,
1759 "metadata": {
1760 "Collapsed": "false"
1761 },
1762 "outputs": [],
1763 "source": [
1764 "data_since_threshold.loc[(slice(None), ['UK']), :].tail(20)"
1765 ]
1766 },
1767 {
1768 "cell_type": "code",
1769 "execution_count": null,
1770 "metadata": {
1771 "Collapsed": "false"
1772 },
1773 "outputs": [],
1774 "source": [
1775 "data_since_threshold.loc[(slice(None), ['ES']), :].tail(20)"
1776 ]
1777 },
1778 {
1779 "cell_type": "markdown",
1780 "metadata": {
1781 "Collapsed": "false"
1782 },
1783 "source": [
1784 "## Death projections"
1785 ]
1786 },
1787 {
1788 "cell_type": "code",
1789 "execution_count": null,
1790 "metadata": {
1791 "Collapsed": "false"
1792 },
1793 "outputs": [],
1794 "source": [
1795 "data_since_threshold.loc[(slice(None), ['UK']), :].tail(15)"
1796 ]
1797 },
1798 {
1799 "cell_type": "code",
1800 "execution_count": null,
1801 "metadata": {
1802 "Collapsed": "false"
1803 },
1804 "outputs": [],
1805 "source": [
1806 "it_since_threshold = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['IT']), :]\n",
1807 "s_end = it_since_threshold.index.max()[0]\n",
1808 "s_end"
1809 ]
1810 },
1811 {
1812 "cell_type": "code",
1813 "execution_count": null,
1814 "metadata": {
1815 "Collapsed": "false"
1816 },
1817 "outputs": [],
1818 "source": [
1819 "uk_projection = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), :]\n",
1820 "uk_current_end = uk_projection.index.max()[0] + 1\n",
1821 "# s_start = uk_projection.index.max()[0] + 1\n",
1822 "uk_current_end"
1823 ]
1824 },
1825 {
1826 "cell_type": "code",
1827 "execution_count": null,
1828 "metadata": {
1829 "Collapsed": "false"
1830 },
1831 "outputs": [],
1832 "source": [
1833 "current_uk_deaths_m7 = uk_projection[uk_projection.deaths_m7 >= 0].iloc[-1].deaths_m7\n",
1834 "current_uk_deaths_m7"
1835 ]
1836 },
1837 {
1838 "cell_type": "code",
1839 "execution_count": null,
1840 "metadata": {
1841 "Collapsed": "false"
1842 },
1843 "outputs": [],
1844 "source": [
1845 "it_since_threshold[it_since_threshold.deaths_m7 <= current_uk_deaths_m7].loc[60:].first_valid_index()[0]"
1846 ]
1847 },
1848 {
1849 "cell_type": "code",
1850 "execution_count": null,
1851 "metadata": {
1852 "Collapsed": "false"
1853 },
1854 "outputs": [],
1855 "source": [
1856 "s_start = it_since_threshold[it_since_threshold.deaths_m7 <= current_uk_deaths_m7].loc[60:].first_valid_index()[0]\n",
1857 "s_start"
1858 ]
1859 },
1860 {
1861 "cell_type": "code",
1862 "execution_count": null,
1863 "metadata": {
1864 "Collapsed": "false"
1865 },
1866 "outputs": [],
1867 "source": [
1868 "s_start_date = data_since_threshold.loc[(89, 'IT'), 'dateRep']# .iloc[0]\n",
1869 "s_start_date"
1870 ]
1871 },
1872 {
1873 "cell_type": "code",
1874 "execution_count": null,
1875 "metadata": {
1876 "Collapsed": "false"
1877 },
1878 "outputs": [],
1879 "source": [
1880 "s_end - s_start"
1881 ]
1882 },
1883 {
1884 "cell_type": "code",
1885 "execution_count": null,
1886 "metadata": {
1887 "Collapsed": "false"
1888 },
1889 "outputs": [],
1890 "source": [
1891 "uk_end = s_end - s_start + uk_current_end\n",
1892 "uk_end"
1893 ]
1894 },
1895 {
1896 "cell_type": "code",
1897 "execution_count": null,
1898 "metadata": {
1899 "Collapsed": "false"
1900 },
1901 "outputs": [],
1902 "source": [
1903 "proj = it_since_threshold.loc[(slice(s_start, s_end), slice(None)), ['cases', 'deaths', 'deaths_m7']]\n",
1904 "ndiff = uk_current_end - s_start\n",
1905 "proj.index = pd.MultiIndex.from_tuples([(n + ndiff, 'UK') for n, _ in proj.index], names=proj.index.names)\n",
1906 "proj"
1907 ]
1908 },
1909 {
1910 "cell_type": "code",
1911 "execution_count": null,
1912 "metadata": {
1913 "Collapsed": "false"
1914 },
1915 "outputs": [],
1916 "source": [
1917 "it_since_threshold.loc[(slice(s_start - 8, s_start + 2), slice(None)), ['cases', 'deaths', 'deaths_m7']]"
1918 ]
1919 },
1920 {
1921 "cell_type": "code",
1922 "execution_count": null,
1923 "metadata": {
1924 "Collapsed": "false"
1925 },
1926 "outputs": [],
1927 "source": [
1928 "uk_projection[['cases', 'deaths', 'deaths_m7']].tail()"
1929 ]
1930 },
1931 {
1932 "cell_type": "code",
1933 "execution_count": null,
1934 "metadata": {
1935 "Collapsed": "false"
1936 },
1937 "outputs": [],
1938 "source": [
1939 "# proj['deaths_m7'] = proj['deaths_m7'] + 20\n",
1940 "# proj"
1941 ]
1942 },
1943 {
1944 "cell_type": "markdown",
1945 "metadata": {
1946 "Collapsed": "false"
1947 },
1948 "source": [
1949 "Projected deaths, UK following IT trend from now."
1950 ]
1951 },
1952 {
1953 "cell_type": "code",
1954 "execution_count": null,
1955 "metadata": {
1956 "Collapsed": "false"
1957 },
1958 "outputs": [],
1959 "source": [
1960 "uk_projection = uk_projection.append(proj, sort=True)\n",
1961 "uk_projection.deaths.sum()"
1962 ]
1963 },
1964 {
1965 "cell_type": "code",
1966 "execution_count": null,
1967 "metadata": {
1968 "Collapsed": "false"
1969 },
1970 "outputs": [],
1971 "source": [
1972 "uk_projection = uk_projection.droplevel(1)\n",
1973 "uk_projection"
1974 ]
1975 },
1976 {
1977 "cell_type": "code",
1978 "execution_count": null,
1979 "metadata": {
1980 "Collapsed": "false"
1981 },
1982 "outputs": [],
1983 "source": [
1984 "uk_projection.loc[152, 'deaths']"
1985 ]
1986 },
1987 {
1988 "cell_type": "markdown",
1989 "metadata": {
1990 "Collapsed": "false"
1991 },
1992 "source": [
1993 "## Correction for cumulative deaths correction on 14 August"
1994 ]
1995 },
1996 {
1997 "cell_type": "code",
1998 "execution_count": null,
1999 "metadata": {
2000 "Collapsed": "false"
2001 },
2002 "outputs": [],
2003 "source": [
2004 "# uk_projection.loc[152, 'deaths'] = 50"
2005 ]
2006 },
2007 {
2008 "cell_type": "code",
2009 "execution_count": null,
2010 "metadata": {
2011 "Collapsed": "false"
2012 },
2013 "outputs": [],
2014 "source": [
2015 "uk_projection['deaths_m7'] = uk_projection['deaths'].transform(lambda x: x.rolling(7, 1).mean())\n",
2016 "uk_projection.loc[(uk_current_end - 20):(uk_current_end + 5)]"
2017 ]
2018 },
2019 {
2020 "cell_type": "code",
2021 "execution_count": null,
2022 "metadata": {
2023 "Collapsed": "false"
2024 },
2025 "outputs": [],
2026 "source": [
2027 "uk_projection.loc[(uk_current_end - 5):]"
2028 ]
2029 },
2030 {
2031 "cell_type": "code",
2032 "execution_count": null,
2033 "metadata": {
2034 "Collapsed": "false"
2035 },
2036 "outputs": [],
2037 "source": [
2038 "uk_projection.deaths_m7.plot()"
2039 ]
2040 },
2041 {
2042 "cell_type": "code",
2043 "execution_count": null,
2044 "metadata": {
2045 "Collapsed": "false"
2046 },
2047 "outputs": [],
2048 "source": [
2049 "proj.droplevel(level=1)"
2050 ]
2051 },
2052 {
2053 "cell_type": "code",
2054 "execution_count": null,
2055 "metadata": {
2056 "Collapsed": "false"
2057 },
2058 "outputs": [],
2059 "source": [
2060 "ax = deaths_m7[COUNTRIES_CORE].plot()\n",
2061 "# uk_projection['deaths_m7'].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\", label=\"Projection\", style='--', ax=ax)\n",
2062 "proj.droplevel(level=1)['deaths_m7'].plot(figsize=(10, 6), title=\"Deaths per day, 7 day moving average\", label=\"Projection\", style='--', ax=ax)\n",
2063 "ax.set_xlabel(f\"Days since {DEATH_COUNT_THRESHOLD} deaths\")\n",
2064 "for c in COUNTRIES_CORE:\n",
2065 " lvi = deaths_m7[c].last_valid_index()\n",
2066 " ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = c)\n",
2067 "# plt.savefig('covid_deaths_per_day_7.png') "
2068 ]
2069 },
2070 {
2071 "cell_type": "code",
2072 "execution_count": null,
2073 "metadata": {
2074 "Collapsed": "false"
2075 },
2076 "outputs": [],
2077 "source": [
2078 "it_since_threshold.deaths.sum()"
2079 ]
2080 },
2081 {
2082 "cell_type": "markdown",
2083 "metadata": {
2084 "Collapsed": "false"
2085 },
2086 "source": [
2087 "# Excess deaths calculation"
2088 ]
2089 },
2090 {
2091 "cell_type": "code",
2092 "execution_count": null,
2093 "metadata": {
2094 "Collapsed": "false"
2095 },
2096 "outputs": [],
2097 "source": [
2098 "with open('excess_deaths.json') as f:\n",
2099 " excess_deaths_data = json.load(f)\n",
2100 " \n",
2101 "# with open('excess_death_accuracy.json') as f:\n",
2102 "# excess_death_accuracy = json.load(f)\n",
2103 " \n",
2104 "excess_deaths_data"
2105 ]
2106 },
2107 {
2108 "cell_type": "code",
2109 "execution_count": null,
2110 "metadata": {
2111 "Collapsed": "false"
2112 },
2113 "outputs": [],
2114 "source": [
2115 "additional_deaths = data_by_date.loc[('UK', excess_deaths_data['end_date']):('UK')].iloc[1:].deaths.sum()\n",
2116 "additional_deaths"
2117 ]
2118 },
2119 {
2120 "cell_type": "code",
2121 "execution_count": null,
2122 "metadata": {
2123 "Collapsed": "false"
2124 },
2125 "outputs": [],
2126 "source": [
2127 "uk_covid_deaths = data_by_date.loc['UK'].deaths.sum()\n",
2128 "uk_covid_deaths"
2129 ]
2130 },
2131 {
2132 "cell_type": "code",
2133 "execution_count": null,
2134 "metadata": {
2135 "Collapsed": "false"
2136 },
2137 "outputs": [],
2138 "source": [
2139 "uk_deaths_to_date = int(excess_deaths_data['excess_deaths']) + additional_deaths\n",
2140 "uk_deaths_to_date"
2141 ]
2142 },
2143 {
2144 "cell_type": "code",
2145 "execution_count": null,
2146 "metadata": {
2147 "Collapsed": "false"
2148 },
2149 "outputs": [],
2150 "source": [
2151 "# excess_deaths_upto = '2020-05-08'\n",
2152 "# excess_deaths = 54500"
2153 ]
2154 },
2155 {
2156 "cell_type": "code",
2157 "execution_count": null,
2158 "metadata": {
2159 "Collapsed": "false"
2160 },
2161 "outputs": [],
2162 "source": [
2163 "# excess_deaths_upto = excess_deaths_data['end_date']\n",
2164 "# excess_deaths = excess_deaths_data['excess_deaths']"
2165 ]
2166 },
2167 {
2168 "cell_type": "markdown",
2169 "metadata": {
2170 "Collapsed": "false"
2171 },
2172 "source": [
2173 "Recorded deaths in period where ONS has reported total deaths"
2174 ]
2175 },
2176 {
2177 "cell_type": "code",
2178 "execution_count": null,
2179 "metadata": {
2180 "Collapsed": "false"
2181 },
2182 "outputs": [],
2183 "source": [
2184 "# ons_reported_deaths = base_data.loc['UK'][:excess_deaths_upto]['deaths'].sum()\n",
2185 "# ons_reported_deaths"
2186 ]
2187 },
2188 {
2189 "cell_type": "code",
2190 "execution_count": null,
2191 "metadata": {
2192 "Collapsed": "false"
2193 },
2194 "outputs": [],
2195 "source": [
2196 "# excess_deaths_upto"
2197 ]
2198 },
2199 {
2200 "cell_type": "markdown",
2201 "metadata": {
2202 "Collapsed": "false"
2203 },
2204 "source": [
2205 "## Correction for deaths total correction on 14 August"
2206 ]
2207 },
2208 {
2209 "cell_type": "code",
2210 "execution_count": null,
2211 "metadata": {
2212 "Collapsed": "false"
2213 },
2214 "outputs": [],
2215 "source": [
2216 "# ons_unreported_deaths_data = base_data.loc['UK'][excess_deaths_upto:].iloc[1:]['deaths']\n",
2217 "# ons_unreported_deaths_data['2020-08-14'] = 50"
2218 ]
2219 },
2220 {
2221 "cell_type": "code",
2222 "execution_count": null,
2223 "metadata": {
2224 "Collapsed": "false"
2225 },
2226 "outputs": [],
2227 "source": [
2228 "# ons_unreported_deaths = ons_unreported_deaths_data.sum()\n",
2229 "# ons_unreported_deaths"
2230 ]
2231 },
2232 {
2233 "cell_type": "code",
2234 "execution_count": null,
2235 "metadata": {
2236 "Collapsed": "false"
2237 },
2238 "outputs": [],
2239 "source": [
2240 "# scaled_ons_unreported_deaths = ons_unreported_deaths * excess_death_accuracy\n",
2241 "# scaled_ons_unreported_deaths"
2242 ]
2243 },
2244 {
2245 "cell_type": "code",
2246 "execution_count": null,
2247 "metadata": {
2248 "Collapsed": "false"
2249 },
2250 "outputs": [],
2251 "source": [
2252 "# uk_deaths_to_date = excess_deaths + scaled_ons_unreported_deaths\n",
2253 "# uk_deaths_to_date"
2254 ]
2255 },
2256 {
2257 "cell_type": "code",
2258 "execution_count": null,
2259 "metadata": {
2260 "Collapsed": "false"
2261 },
2262 "outputs": [],
2263 "source": [
2264 "# data_since_threshold.loc[(slice(None), 'UK'), :][data_since_threshold.dateRep == excess_deaths_data['end_date']]"
2265 ]
2266 },
2267 {
2268 "cell_type": "code",
2269 "execution_count": null,
2270 "metadata": {
2271 "Collapsed": "false"
2272 },
2273 "outputs": [],
2274 "source": [
2275 "# data_since_threshold[data_since_threshold.dateRep == excess_deaths_data['end_date']].loc[(slice(None), 'UK'), :]"
2276 ]
2277 },
2278 {
2279 "cell_type": "code",
2280 "execution_count": null,
2281 "metadata": {
2282 "Collapsed": "false"
2283 },
2284 "outputs": [],
2285 "source": [
2286 "# ons_unreported_start = data_since_threshold[data_since_threshold.dateRep == excess_deaths_data['end_date']].loc[(slice(None), 'UK'), :].first_valid_index()[0] + 1\n",
2287 "# ons_unreported_start"
2288 ]
2289 },
2290 {
2291 "cell_type": "code",
2292 "execution_count": null,
2293 "metadata": {
2294 "Collapsed": "false"
2295 },
2296 "outputs": [],
2297 "source": [
2298 "# unreported_projected_deaths = uk_projection.loc[ons_unreported_start:].deaths.sum()\n",
2299 "# unreported_projected_deaths"
2300 ]
2301 },
2302 {
2303 "cell_type": "code",
2304 "execution_count": null,
2305 "metadata": {
2306 "Collapsed": "false"
2307 },
2308 "outputs": [],
2309 "source": [
2310 "# unreported_projected_deaths_scaled = unreported_projected_deaths * excess_death_accuracy\n",
2311 "# unreported_projected_deaths_scaled"
2312 ]
2313 },
2314 {
2315 "cell_type": "code",
2316 "execution_count": null,
2317 "metadata": {
2318 "Collapsed": "false"
2319 },
2320 "outputs": [],
2321 "source": [
2322 "# uk_projection.loc[(s_start):].deaths.sum()"
2323 ]
2324 },
2325 {
2326 "cell_type": "code",
2327 "execution_count": null,
2328 "metadata": {
2329 "Collapsed": "false"
2330 },
2331 "outputs": [],
2332 "source": [
2333 "# deaths_actual_projected_scaled = uk_deaths_to_date + uk_projection.loc[(s_start):].deaths.sum() * excess_death_accuracy\n",
2334 "# deaths_actual_projected_scaled"
2335 ]
2336 },
2337 {
2338 "cell_type": "code",
2339 "execution_count": null,
2340 "metadata": {
2341 "Collapsed": "false"
2342 },
2343 "outputs": [],
2344 "source": [
2345 "# excess_deaths / reported_deaths"
2346 ]
2347 },
2348 {
2349 "cell_type": "markdown",
2350 "metadata": {
2351 "Collapsed": "false"
2352 },
2353 "source": [
2354 "True deaths to date, if we follow the scaling of excess deaths over reported deaths so far."
2355 ]
2356 },
2357 {
2358 "cell_type": "code",
2359 "execution_count": null,
2360 "metadata": {
2361 "Collapsed": "false"
2362 },
2363 "outputs": [],
2364 "source": [
2365 "# uk_covid_deaths = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), 'deaths_culm'].iloc[-1]\n",
2366 "# uk_covid_deaths"
2367 ]
2368 },
2369 {
2370 "cell_type": "code",
2371 "execution_count": null,
2372 "metadata": {
2373 "Collapsed": "false"
2374 },
2375 "outputs": [],
2376 "source": [
2377 "# uk_covid_deaths_scaled = excess_deaths + unreported_deaths * excess_death_accuracy\n",
2378 "# uk_covid_deaths_scaled"
2379 ]
2380 },
2381 {
2382 "cell_type": "code",
2383 "execution_count": null,
2384 "metadata": {
2385 "Collapsed": "false"
2386 },
2387 "outputs": [],
2388 "source": [
2389 "# data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['IT']), 'dateRep'].iloc[-1] + pd.Timedelta(s_end - s_start, unit='days')"
2390 ]
2391 },
2392 {
2393 "cell_type": "code",
2394 "execution_count": null,
2395 "metadata": {
2396 "Collapsed": "false"
2397 },
2398 "outputs": [],
2399 "source": [
2400 "# data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), 'dateRep'].iloc[-1].strftime(\"%Y-%m-%d\")"
2401 ]
2402 },
2403 {
2404 "cell_type": "code",
2405 "execution_count": null,
2406 "metadata": {
2407 "Collapsed": "false"
2408 },
2409 "outputs": [],
2410 "source": [
2411 "# uk_covid_deaths * excess_deaths / reported_deaths"
2412 ]
2413 },
2414 {
2415 "cell_type": "code",
2416 "execution_count": null,
2417 "metadata": {
2418 "Collapsed": "false"
2419 },
2420 "outputs": [],
2421 "source": [
2422 "# uk_projection.deaths.sum() * excess_deaths / reported_deaths"
2423 ]
2424 },
2425 {
2426 "cell_type": "code",
2427 "execution_count": null,
2428 "metadata": {
2429 "Collapsed": "false"
2430 },
2431 "outputs": [],
2432 "source": [
2433 "# data_since_threshold.loc[(slice(None), 'FR'), :]\n",
2434 "# data_since_threshold[data_since_threshold.dateRep == '2020-05-18'].loc[(slice(None), 'FR'), :]"
2435 ]
2436 },
2437 {
2438 "cell_type": "markdown",
2439 "metadata": {
2440 "Collapsed": "false"
2441 },
2442 "source": [
2443 "## School reopenings"
2444 ]
2445 },
2446 {
2447 "cell_type": "code",
2448 "execution_count": null,
2449 "metadata": {
2450 "Collapsed": "false"
2451 },
2452 "outputs": [],
2453 "source": [
2454 "school_reopenings = {\n",
2455 " 'ES': {'date': '2020-05-18'},\n",
2456 " 'FR': {'date': '2020-05-18'}, # some areas only\n",
2457 "# 'IT': {'date': '2020-09-01'},\n",
2458 " # 'IE': {'date': '2020-09-01'},\n",
2459 " 'DE': {'date': '2020-05-04'},\n",
2460 " 'UK': {'date': '2020-06-01'}\n",
2461 "}"
2462 ]
2463 },
2464 {
2465 "cell_type": "code",
2466 "execution_count": null,
2467 "metadata": {
2468 "Collapsed": "false"
2469 },
2470 "outputs": [],
2471 "source": [
2472 "data_since_threshold[data_since_threshold.dateRep == '2020-05-04'].loc[(slice(None), ['DE']), :].first_valid_index()"
2473 ]
2474 },
2475 {
2476 "cell_type": "code",
2477 "execution_count": null,
2478 "metadata": {
2479 "Collapsed": "false"
2480 },
2481 "outputs": [],
2482 "source": [
2483 "data_since_threshold[data_since_threshold.dateRep == '2020-05-04'].loc[(slice(None), ['DE']), :].iloc[0].deaths_m7"
2484 ]
2485 },
2486 {
2487 "cell_type": "code",
2488 "execution_count": null,
2489 "metadata": {
2490 "Collapsed": "false"
2491 },
2492 "outputs": [],
2493 "source": [
2494 "for cID in school_reopenings:\n",
2495 " dst_in = data_since_threshold[data_since_threshold.dateRep == (school_reopenings[cID]['date'])].loc[(slice(None), [cID]), :]\n",
2496 " dst_i = dst_in.first_valid_index()\n",
2497 " dst_n = dst_in.iloc[0].deaths_m7\n",
2498 " school_reopenings[cID]['since_threshold'] = dst_i[0]\n",
2499 " school_reopenings[cID]['deaths_m7'] = dst_n\n",
2500 "school_reopenings"
2501 ]
2502 },
2503 {
2504 "cell_type": "code",
2505 "execution_count": null,
2506 "metadata": {
2507 "Collapsed": "false"
2508 },
2509 "outputs": [],
2510 "source": [
2511 "ax = deaths_m7[COUNTRIES_CORE].plot(figsize=(15, 9), title=\"Deaths per day, 7 day moving average\")\n",
2512 "# uk_projection.deaths_m7.plot(ax=ax)\n",
2513 "for c in COUNTRIES_CORE:\n",
2514 " lvi = deaths_m7[c].last_valid_index()\n",
2515 " ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = f\"{c}: {deaths_m7[c][lvi]:.0f}\")\n",
2516 " if c in school_reopenings:\n",
2517 " marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()\n",
2518 " ax.plot(school_reopenings[c]['since_threshold'], school_reopenings[c]['deaths_m7'], '*', \n",
2519 " markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)\n",
2520 " ax.text(x = school_reopenings[c]['since_threshold'] + 1, y = school_reopenings[c]['deaths_m7'], \n",
2521 " s = f\"{school_reopenings[c]['date']}: {school_reopenings[c]['deaths_m7']:.0f}\")\n",
2522 "plt.savefig('school_reopenings.png')"
2523 ]
2524 },
2525 {
2526 "cell_type": "code",
2527 "execution_count": null,
2528 "metadata": {
2529 "Collapsed": "false"
2530 },
2531 "outputs": [],
2532 "source": [
2533 "# ax = deaths_m7[COUNTRIES_CORE].plot(figsize=(15, 9), title=\"Deaths per day, 7 day moving average\",\n",
2534 "# xlim=(46, 91), ylim=(0, 400))\n",
2535 "# # uk_projection.deaths_m7.plot(ax=ax)\n",
2536 "# for c in COUNTRIES_CORE:\n",
2537 "# lvi = deaths_m7[c].last_valid_index()\n",
2538 "# ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = f\"{c}: {deaths_m7[c][lvi]:.0f}\", fontsize=14)\n",
2539 "# if c in school_reopenings:\n",
2540 "# marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()\n",
2541 "# ax.plot(school_reopenings[c]['since_threshold'], school_reopenings[c]['deaths_m7'], '*', \n",
2542 "# markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)\n",
2543 "# ax.text(x = school_reopenings[c]['since_threshold'] + 1, y = school_reopenings[c]['deaths_m7'], \n",
2544 "# s = f\"{school_reopenings[c]['date']}: {school_reopenings[c]['deaths_m7']:.0f}\",\n",
2545 "# fontsize=14)\n",
2546 "# plt.savefig('school_reopenings_detail.png')"
2547 ]
2548 },
2549 {
2550 "cell_type": "markdown",
2551 "metadata": {
2552 "Collapsed": "false"
2553 },
2554 "source": [
2555 "# Lockdown graphs"
2556 ]
2557 },
2558 {
2559 "cell_type": "code",
2560 "execution_count": null,
2561 "metadata": {
2562 "Collapsed": "false"
2563 },
2564 "outputs": [],
2565 "source": [
2566 "lockdown_dates = {\n",
2567 " 'ES': { 'part_start': {'date': '2020-03-14'}\n",
2568 " , 'full_start': {'date': '2020-03-15'}\n",
2569 " , 'part_finish': {'date': '2020-05-18'}\n",
2570 " },\n",
2571 " 'FR': { 'part_start': {'date': '2020-03-13'}\n",
2572 " , 'full_start': {'date': '2020-03-17'}\n",
2573 " , 'part_finish': {'date': '2020-05-11'}\n",
2574 " },\n",
2575 " 'IT': { 'part_start': {'date': '2020-03-08'}\n",
2576 " , 'full_start': {'date': '2020-03-10'}\n",
2577 " , 'part_finish': {'date': '2020-05-04'}\n",
2578 " },\n",
2579 " 'DE': { #'part_start': {'date': '2020-03-13'}\n",
2580 " 'full_start': {'date': '2020-03-22'}\n",
2581 " , 'part_finish': {'date': '2020-05-06'}\n",
2582 " },\n",
2583 " 'UK': { 'part_start': {'date': '2020-03-23'}\n",
2584 " , 'full_start': {'date': '2020-03-23'}\n",
2585 " , 'part_finish': {'date': '2020-05-31'}\n",
2586 " },\n",
2587 " 'IE': { #'part_start': {'date': '2020-03-12'}\n",
2588 " 'full_start': {'date': '2020-03-27'}\n",
2589 " , 'part_finish': {'date': '2020-05-18'}\n",
2590 " },\n",
2591 "}"
2592 ]
2593 },
2594 {
2595 "cell_type": "code",
2596 "execution_count": null,
2597 "metadata": {
2598 "Collapsed": "false"
2599 },
2600 "outputs": [],
2601 "source": [
2602 "for cID in lockdown_dates:\n",
2603 " for phase in lockdown_dates[cID]:\n",
2604 " dst_in = data_since_threshold[data_since_threshold.dateRep == (lockdown_dates[cID][phase]['date'])].loc[(slice(None), [cID]), :]\n",
2605 " dst_i = dst_in.first_valid_index()\n",
2606 " dst_n = dst_in.iloc[0].deaths_m7\n",
2607 " dst_c = dst_in.iloc[0].cases_m7\n",
2608 " lockdown_dates[cID][phase]['since_threshold'] = dst_i[0]\n",
2609 " lockdown_dates[cID][phase]['deaths_m7'] = dst_n\n",
2610 " lockdown_dates[cID][phase]['cases_m7'] = dst_c\n",
2611 "\n",
2612 "lockdown_dates"
2613 ]
2614 },
2615 {
2616 "cell_type": "code",
2617 "execution_count": null,
2618 "metadata": {
2619 "Collapsed": "false"
2620 },
2621 "outputs": [],
2622 "source": [
2623 "ax = deaths_m7[COUNTRIES_CORE].plot(figsize=(15, 9), title=\"Deaths per day, 7 day moving averagee, with lockdown dates\")\n",
2624 "ax.set_xlabel(f\"Days since {DEATH_COUNT_THRESHOLD} deaths\")\n",
2625 "# uk_projection.deaths_m7.plot(ax=ax)\n",
2626 "for c in COUNTRIES_CORE:\n",
2627 " lvi = deaths_m7[c].last_valid_index()\n",
2628 " if c != 'UK':\n",
2629 " ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = f\"{c}: {deaths_m7[c][lvi]:.0f}\")\n",
2630 " if c in lockdown_dates:\n",
2631 " for phase in lockdown_dates[c]:\n",
2632 " marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()\n",
2633 " ax.plot(lockdown_dates[c][phase]['since_threshold'], lockdown_dates[c][phase]['deaths_m7'], '*',\n",
2634 " markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)\n",
2635 " if 'start' not in phase:\n",
2636 " ax.text(x = lockdown_dates[c][phase]['since_threshold'] + 1, y = lockdown_dates[c][phase]['deaths_m7'], \n",
2637 " s = f\"{lockdown_dates[c][phase]['date']}: {lockdown_dates[c][phase]['deaths_m7']:.0f}\")\n",
2638 "# plt.savefig('school_reopenings.png')"
2639 ]
2640 },
2641 {
2642 "cell_type": "code",
2643 "execution_count": null,
2644 "metadata": {
2645 "Collapsed": "false"
2646 },
2647 "outputs": [],
2648 "source": [
2649 "ax = cases_m7.iloc[-50:][COUNTRIES_CORE].plot(figsize=(15, 9), title=\"Cases per day, 7 day moving average, with lockdown dates\") #, ylim=(-10, 1500))\n",
2650 "ax.set_xlabel(f\"Days since {DEATH_COUNT_THRESHOLD} deaths\")\n",
2651 "# uk_projection.deaths_m7.plot(ax=ax)\n",
2652 "for c in COUNTRIES_CORE:\n",
2653 " lvi = cases_m7[c].last_valid_index()\n",
2654 "# if c != 'UK':\n",
2655 " ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = f\"{c}: {cases_m7[c][lvi]:.0f}\")\n"
2656 ]
2657 },
2658 {
2659 "cell_type": "code",
2660 "execution_count": null,
2661 "metadata": {
2662 "Collapsed": "false"
2663 },
2664 "outputs": [],
2665 "source": [
2666 "ax = cases_m7[COUNTRIES_CORE].plot(figsize=(15, 9), title=\"Cases per day, 7 day moving average, with lockdown dates\")\n",
2667 "ax.set_xlabel(f\"Days since {DEATH_COUNT_THRESHOLD} deaths\")\n",
2668 "# uk_projection.deaths_m7.plot(ax=ax)\n",
2669 "for c in COUNTRIES_CORE:\n",
2670 " lvi = cases_m7[c].last_valid_index()\n",
2671 "# if c != 'UK':\n",
2672 " ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = f\"{c}: {cases_m7[c][lvi]:.0f}\")\n",
2673 " if c in lockdown_dates:\n",
2674 " for phase in lockdown_dates[c]:\n",
2675 " marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()\n",
2676 " if 'start' in phase:\n",
2677 " marker_shape = '^'\n",
2678 " else:\n",
2679 " marker_shape = 'v'\n",
2680 " ax.plot(lockdown_dates[c][phase]['since_threshold'], lockdown_dates[c][phase]['cases_m7'], \n",
2681 " marker_shape,\n",
2682 " markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)\n",
2683 " if 'start' not in phase:\n",
2684 " ax.text(x = lockdown_dates[c][phase]['since_threshold'] + 1, y = lockdown_dates[c][phase]['cases_m7'], \n",
2685 " s = f\"{lockdown_dates[c][phase]['date']}: {lockdown_dates[c][phase]['cases_m7']:.0f}\")\n",
2686 "# plt.savefig('cases_per_day_with_lockdown.png')"
2687 ]
2688 },
2689 {
2690 "cell_type": "code",
2691 "execution_count": null,
2692 "metadata": {
2693 "Collapsed": "false"
2694 },
2695 "outputs": [],
2696 "source": [
2697 "plot_start_date = '2020-03-01'\n",
2698 "ax = cases_by_date_m7.loc[plot_start_date:, COUNTRIES_CORE].plot(figsize=(15, 9), title=\"Cases per day, 7 day moving average, with lockdown dates\")\n",
2699 "ax.set_xlabel(f\"Date\")\n",
2700 "ax.set_ylabel(\"Number of cases\")\n",
2701 "# uk_projection.deaths_m7.plot(ax=ax)\n",
2702 "for c in COUNTRIES_CORE:\n",
2703 " lvi = cases_by_date_m7[c].last_valid_index()\n",
2704 "# if c != 'UK':\n",
2705 " ax.text(x = lvi + pd.Timedelta(days=1), y = cases_by_date_m7[c][lvi], s = f\"{c}: {cases_by_date_m7[c][lvi]:.0f}\")\n",
2706 " if c in lockdown_dates:\n",
2707 " for phase in lockdown_dates[c]:\n",
2708 " marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()\n",
2709 " if 'start' in phase:\n",
2710 " marker_shape = '^'\n",
2711 " else:\n",
2712 " marker_shape = 'v'\n",
2713 " marker_x_pos = ax.get_xlim()[0] + mpl.dates.date2num(pd.to_datetime(lockdown_dates[c][phase]['date'])) - mpl.dates.date2num(pd.to_datetime(plot_start_date))\n",
2714 " ax.plot(marker_x_pos, lockdown_dates[c][phase]['cases_m7'], \n",
2715 " marker_shape,\n",
2716 " markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)\n",
2717 " if 'start' not in phase:\n",
2718 " ax.text(x = marker_x_pos + 3, y = lockdown_dates[c][phase]['cases_m7'], \n",
2719 " s = f\"{lockdown_dates[c][phase]['date']}: {lockdown_dates[c][phase]['cases_m7']:.0f}\")\n",
2720 "plt.savefig('cases_per_day_with_lockdown.png')"
2721 ]
2722 },
2723 {
2724 "cell_type": "code",
2725 "execution_count": null,
2726 "metadata": {
2727 "Collapsed": "false"
2728 },
2729 "outputs": [],
2730 "source": [
2731 "ax = cases_m7[COUNTRIES_CORE].plot(figsize=(10, 6), title=\"Cases per day, 7 day moving average\")\n",
2732 "for c in COUNTRIES_CORE:\n",
2733 " lvi = cases_m7[c].last_valid_index()\n",
2734 " ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = c)\n",
2735 "plt.savefig('covid_cases_per_day-core.png') "
2736 ]
2737 },
2738 {
2739 "cell_type": "code",
2740 "execution_count": null,
2741 "metadata": {
2742 "Collapsed": "false"
2743 },
2744 "outputs": [],
2745 "source": [
2746 "ax = deaths_m7[COUNTRIES_CORE].plot(figsize=(15, 9), title=\"Deaths per day, 7 day moving average\",\n",
2747 " xlim=(0, 15), \n",
2748 " ylim=(0, 66)\n",
2749 " )\n",
2750 "# uk_projection.deaths_m7.plot(ax=ax)\n",
2751 "for c in COUNTRIES_CORE:\n",
2752 " lvi = deaths_m7[c].last_valid_index()\n",
2753 " if c in lockdown_dates:\n",
2754 " for phase in lockdown_dates[c]:\n",
2755 " if 'start' in phase:\n",
2756 " print(c, phase)\n",
2757 " marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()\n",
2758 " ax.plot(lockdown_dates[c][phase]['since_threshold'], lockdown_dates[c][phase]['deaths_m7'], '*', \n",
2759 " markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)\n",
2760 " ax.text(x = lockdown_dates[c][phase]['since_threshold'] + 0.3, y = lockdown_dates[c][phase]['deaths_m7'], \n",
2761 " s = f\"{lockdown_dates[c][phase]['date']}: {lockdown_dates[c][phase]['deaths_m7']:.0f}\")\n",
2762 "# plt.savefig('school_reopenings.png')"
2763 ]
2764 },
2765 {
2766 "cell_type": "code",
2767 "execution_count": null,
2768 "metadata": {
2769 "Collapsed": "false"
2770 },
2771 "outputs": [],
2772 "source": []
2773 },
2774 {
2775 "cell_type": "code",
2776 "execution_count": null,
2777 "metadata": {
2778 "Collapsed": "false"
2779 },
2780 "outputs": [],
2781 "source": []
2782 },
2783 {
2784 "cell_type": "markdown",
2785 "metadata": {
2786 "Collapsed": "false"
2787 },
2788 "source": [
2789 "# Write results to summary file"
2790 ]
2791 },
2792 {
2793 "cell_type": "code",
2794 "execution_count": null,
2795 "metadata": {
2796 "Collapsed": "false"
2797 },
2798 "outputs": [],
2799 "source": [
2800 "with open('covid_summary.md', 'w') as f:\n",
2801 " f.write('% Covid death data summary\\n')\n",
2802 " f.write('% Neil Smith\\n')\n",
2803 " f.write(f'% Created on {datetime.datetime.now().strftime(\"%Y-%m-%d\")}\\n')\n",
2804 " f.write('\\n')\n",
2805 " \n",
2806 " last_uk_date = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), 'dateRep'].iloc[-1]\n",
2807 " f.write(f'> Last UK data from {last_uk_date.strftime(\"%Y-%m-%d\")}\\n')\n",
2808 " f.write('\\n') "
2809 ]
2810 },
2811 {
2812 "cell_type": "code",
2813 "execution_count": null,
2814 "metadata": {
2815 "Collapsed": "false"
2816 },
2817 "outputs": [],
2818 "source": [
2819 "with open('covid_summary.md', 'a') as f:\n",
2820 " f.write('## Headlines\\n')\n",
2821 " f.write('\\n')\n",
2822 " f.write('| []() | |\\n')\n",
2823 " f.write('|:---|---:|\\n')\n",
2824 " f.write(f'| Deaths reported so far | {uk_covid_deaths} | \\n')\n",
2825 " f.write(f'| Total Covid deaths to date (estimated) | {uk_deaths_to_date:.0f} |\\n')\n",
2826 " projection_date = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['IT']), 'dateRep'].iloc[-1] + pd.Timedelta(s_end - s_start, unit='days')\n",
2827 "# f.write(f'| Projected total deaths up to {projection_date.strftime(\"%Y-%m-%d\")} | {deaths_actual_projected_scaled:.0f} | \\n')\n",
2828 " f.write('\\n')"
2829 ]
2830 },
2831 {
2832 "cell_type": "code",
2833 "execution_count": null,
2834 "metadata": {
2835 "Collapsed": "false"
2836 },
2837 "outputs": [],
2838 "source": [
2839 "with open('covid_summary.md', 'a') as f:\n",
2840 " f.write('## Total deaths\\n')\n",
2841 "# f.write(f'Time based on days since {DEATH_COUNT_THRESHOLD} deaths\\n')\n",
2842 " f.write('\\n')\n",
2843 " f.write('![Total deaths](covid_deaths_total_linear.png)\\n')\n",
2844 " f.write('\\n')\n",
2845 " f.write('| Country ID | Country name | Total deaths |\\n')\n",
2846 " f.write('|:-----------|:-------------|-------------:|\\n')\n",
2847 " for c in sorted(COUNTRIES_CORE):\n",
2848 " lvi = deaths_by_date[c].last_valid_index()\n",
2849 " f.write(f'| {c} | {countries.loc[c].countriesAndTerritories} | {int(deaths_by_date[c][lvi])} |\\n')\n",
2850 " f.write('\\n')"
2851 ]
2852 },
2853 {
2854 "cell_type": "code",
2855 "execution_count": null,
2856 "metadata": {
2857 "Collapsed": "false"
2858 },
2859 "outputs": [],
2860 "source": [
2861 "with open('covid_summary.md', 'a') as f:\n",
2862 " f.write('## All-causes deaths, UK\\n')\n",
2863 " f.write('\\n')\n",
2864 " f.write('![All-causes deaths](deaths-radar.png)\\n')\n",
2865 " f.write('\\n')\n",
2866 " f.write('### True deaths\\n')\n",
2867 " f.write('\\n')\n",
2868 " f.write(f'The number of deaths reported in official statistics, {uk_covid_deaths}, is an underestimate '\n",
2869 " 'of the true number of Covid deaths.\\n'\n",
2870 " 'This is especially true early in the pandemic, approximately March to May 2020.\\n')\n",
2871 " f.write('We can get a better understanding of the impact of Covid by looking at the number of deaths, '\n",
2872 " 'over and above what would be expected at each week of the year.\\n')\n",
2873 " f.write(f'The ONS (and other bodies in Scotland and Northern Ireland) have released data on the number of deaths '\n",
2874 " f'up to {pd.to_datetime(excess_deaths_data[\"end_date\"]).strftime(\"%d %B %Y\")}.\\n\\n')\n",
2875 " f.write('If, for each of those weeks, I take the largest of the excess deaths or the reported Covid deaths, ')\n",
2876 " f.write(f'I estimate there have been **{uk_deaths_to_date}** total deaths so far.\\n')\n",
2877 " f.write('\\n')"
2878 ]
2879 },
2880 {
2881 "cell_type": "code",
2882 "execution_count": null,
2883 "metadata": {
2884 "Collapsed": "false"
2885 },
2886 "outputs": [],
2887 "source": [
2888 "# with open('covid_summary.md', 'a') as f:\n",
2889 "# f.write(f'In that period, the UK reported {ons_reported_deaths} Covid deaths.\\n')\n",
2890 "# f.write(f'In the last three weeks for which excess deaths have been reported, the excess deaths have been {excess_death_accuracy:.3f} higher than the Covid-reported deaths.\\n')\n",
2891 "# # f.write(f'That means the actual number of Covid death is about {excess_deaths / reported_deaths:.2f} times higher than the reported figures.\\n')\n",
2892 "# f.write('\\n')\n",
2893 "# f.write(f'The UK has reported {uk_covid_deaths} deaths so far.\\n')\n",
2894 "# f.write(f'Using the scaling factor above (for Covid-19 deaths after the ONS figures), I infer that there have been **{uk_deaths_to_date:.0f}** total deaths so far.\\n')\n",
2895 "# f.write('\\n')"
2896 ]
2897 },
2898 {
2899 "cell_type": "code",
2900 "execution_count": null,
2901 "metadata": {
2902 "Collapsed": "false"
2903 },
2904 "outputs": [],
2905 "source": [
2906 "with open('covid_summary.md', 'a') as f:\n",
2907 " f.write('## Deaths per day\\n')\n",
2908 " f.write(f'Based on a 7-day moving average\\n')\n",
2909 " f.write('\\n')\n",
2910 " f.write('![Deaths per day](covid_deaths_per_day_7.png)\\n')\n",
2911 " f.write('\\n')\n",
2912 " f.write('![Deaths per day, last 30 days](deaths_by_date_last_30_days.png)\\n')\n",
2913 " f.write('\\n')"
2914 ]
2915 },
2916 {
2917 "cell_type": "code",
2918 "execution_count": null,
2919 "metadata": {
2920 "Collapsed": "false"
2921 },
2922 "outputs": [],
2923 "source": [
2924 "s_end - s_start - 1"
2925 ]
2926 },
2927 {
2928 "cell_type": "code",
2929 "execution_count": null,
2930 "metadata": {
2931 "Collapsed": "false"
2932 },
2933 "outputs": [],
2934 "source": [
2935 "with open('covid_summary.md', 'a') as f:\n",
2936 " f.write('## Projected deaths\\n')\n",
2937 " f.write(\"Previously, I was using Italy's deaths data to predict the UK's deaths data. \"\n",
2938 " \"This worked when both countries' trends of deaths were falling or constant, \"\n",
2939 " \"as they were until September.\\n\")\n",
2940 " f.write(\"\\n\")\n",
2941 " f.write(\"As of mid-September, with cases rising in both countries at around the same time, \"\n",
2942 " \"I can't use Italian data to predict the UK's future deaths.\\n\")\n",
2943 " f.write(\"\\n\")\n",
2944 "# f.write(f\"The UK's daily deaths data is very similar to Italy's.\\n\")\n",
2945 "# f.write(f'If I use the Italian data for the next {s_end - s_start - 1} days (from {s_start_date.strftime(\"%d %B %Y\")} onwards),')\n",
2946 "# f.write(f' the UK will report {uk_projection.deaths.sum()} deaths on day {uk_end} of the epidemic.\\n')\n",
2947 "# f.write('\\n')\n",
2948 "# f.write('Using the excess deaths scaling from above, that will translate into ')\n",
2949 "# f.write(f'**{deaths_actual_projected_scaled:.0f}** Covid deaths total.\\n')\n",
2950 "# f.write('\\n')"
2951 ]
2952 },
2953 {
2954 "cell_type": "code",
2955 "execution_count": null,
2956 "metadata": {
2957 "Collapsed": "false"
2958 },
2959 "outputs": [],
2960 "source": [
2961 "with open('covid_summary.md', 'a') as f:\n",
2962 " f.write('## Deaths doubling times\\n')\n",
2963 " f.write(f'Based on a 7-day moving average\\n')\n",
2964 " f.write('\\n')\n",
2965 " f.write('![Deaths doubling times](covid_doubling_times_7.png)\\n')\n",
2966 " f.write('\\n')"
2967 ]
2968 },
2969 {
2970 "cell_type": "code",
2971 "execution_count": null,
2972 "metadata": {
2973 "Collapsed": "false"
2974 },
2975 "outputs": [],
2976 "source": [
2977 "with open('covid_summary.md', 'a') as f:\n",
2978 " f.write('\\n')\n",
2979 " f.write('## Cases per day and lockdown dates\\n')\n",
2980 " f.write(f'Based on a 7-day moving average\\n')\n",
2981 " f.write('\\n')\n",
2982 " f.write('![Cases per day](cases_per_day_with_lockdown.png)\\n')\n",
2983 " f.write('\\n')"
2984 ]
2985 },
2986 {
2987 "cell_type": "code",
2988 "execution_count": null,
2989 "metadata": {
2990 "Collapsed": "false"
2991 },
2992 "outputs": [],
2993 "source": [
2994 "with open('covid_summary.md', 'a') as f:\n",
2995 " f.write('| Country ID | Country name | Most recent daily cases | Most recent daily deaths |\\n')\n",
2996 " f.write('|:-----------|:-------------|------------------------:|-------------------------:|\\n')\n",
2997 " for c in sorted(COUNTRIES_CORE):\n",
2998 " lvic = cases_m7[c].last_valid_index()\n",
2999 " lvid = deaths_m7[c].last_valid_index()\n",
3000 " f.write(f'| {c} | {countries.loc[c].countriesAndTerritories} | {cases_m7[c][lvic]:.0f} | {deaths_m7[c][lvid]:.0f} | \\n')\n",
3001 " f.write('\\n')\n",
3002 " f.write('(Figures are 7-day averages)\\n')\n",
3003 " f.write('\\n')"
3004 ]
3005 },
3006 {
3007 "cell_type": "code",
3008 "execution_count": null,
3009 "metadata": {
3010 "Collapsed": "false"
3011 },
3012 "outputs": [],
3013 "source": [
3014 "with open('hospital_normalisation_date.json') as f:\n",
3015 " hospital_normalisation_date_data = json.load(f)"
3016 ]
3017 },
3018 {
3019 "cell_type": "code",
3020 "execution_count": null,
3021 "metadata": {
3022 "Collapsed": "false"
3023 },
3024 "outputs": [],
3025 "source": [
3026 "with open('covid_summary.md', 'a') as f:\n",
3027 " f.write('## Hospital care\\n')\n",
3028 " f.write(f'Based on a 7-day moving average\\n')\n",
3029 " f.write('\\n')\n",
3030 " f.write('![Cases, admissions, deaths](cases_admissions_deaths.png)\\n')\n",
3031 " f.write('\\n')\n",
3032 "# f.write('Admissions are shifted by 10 days, deaths by 25 days. '\n",
3033 "# 'This reflects the typical timescales of infection: '\n",
3034 "# 'patients are admitted 10 days after onset of symptoms, '\n",
3035 "# 'and die 15 days after admission.\\n')\n",
3036 "# f.write('\\n')\n",
3037 "# f.write('Plotting this data with offsets shows more clearly '\n",
3038 "# 'the relative changes in these three metrics.\\n')\n",
3039 " f.write('Due to the large scale differences between the three '\n",
3040 " 'measures, they are all normalised to show changes ')\n",
3041 " f.write(f'since {pd.to_datetime(hospital_normalisation_date_data[\"hospital_normalisation_date\"]).strftime(\"%d %B %Y\")}.\\n')\n",
3042 " f.write('\\n')"
3043 ]
3044 },
3045 {
3046 "cell_type": "code",
3047 "execution_count": null,
3048 "metadata": {
3049 "Collapsed": "false"
3050 },
3051 "outputs": [],
3052 "source": [
3053 "with open('covid_summary.md', 'a') as f:\n",
3054 " f.write('## Testing effectiveness\\n')\n",
3055 " f.write('\\n')\n",
3056 " f.write('A question about testing is whether more detected cases is a result of more tests being '\n",
3057 " 'done or is because the number of cases is increasing. One way of telling the differeence '\n",
3058 " 'is by looking at the fraction of tests that are positive.\\n')\n",
3059 " f.write('\\n')\n",
3060 " f.write('![Positive tests and cases](tests_and_cases.png)\\n')\n",
3061 " f.write('\\n')\n",
3062 " f.write('Numbers of positive tests and cases, '\n",
3063 " '7-day moving average.\\n'\n",
3064 " 'Note the different y-axes\\n')\n",
3065 " f.write('\\n') \n",
3066 " f.write('![Fraction of tests with positive result](fraction_positive_tests.png)\\n')\n",
3067 " f.write('\\n')\n",
3068 " f.write('Fraction of tests with a positive result, both daily figures and '\n",
3069 " '7-day moving average.\\n')\n",
3070 " f.write('\\n') \n",
3071 " f.write('\\n')\n",
3072 " f.write('![Tests against fraction positive, trajectory](fraction_positive_tests_vs_tests.png)\\n')\n",
3073 " f.write('\\n')\n",
3074 " f.write('The trajectory of tests done vs fraction positive tests.\\n')\n",
3075 " f.write('\\n')\n",
3076 " f.write('Points higher indicate more tests; points to the right indicate more positive tests.'\n",
3077 " 'More tests being done with the same infection prevelance will move the point up '\n",
3078 " 'and to the left.\\n')\n",
3079 " f.write('\\n')\n",
3080 " f.write('\\n')\n",
3081 " f.write('![Tests against fraction positive, trajectory](tests_vs_fraction_positive_animation.png)\\n')\n",
3082 " f.write('\\n')"
3083 ]
3084 },
3085 {
3086 "cell_type": "code",
3087 "execution_count": null,
3088 "metadata": {
3089 "Collapsed": "false"
3090 },
3091 "outputs": [],
3092 "source": []
3093 },
3094 {
3095 "cell_type": "code",
3096 "execution_count": null,
3097 "metadata": {
3098 "Collapsed": "false"
3099 },
3100 "outputs": [],
3101 "source": [
3102 "with open('covid_summary.md', 'a') as f:\n",
3103 " f.write('# Data sources\\n')\n",
3104 " f.write('\\n')\n",
3105 " f.write('> Covid data from [European Centre for Disease Prevention and Control](https://www.ecdc.europa.eu/en/publications-data/download-todays-data-geographic-distribution-covid-19-cases-worldwide)\\n')\n",
3106 " f.write('\\n') \n",
3107 " f.write(\"\"\"> Population data from:\n",
3108 "\n",
3109 "* [Office of National Statistics](https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/deaths/datasets/weeklyprovisionalfiguresondeathsregisteredinenglandandwales) (Endland and Wales) Weeks start on a Saturday.\n",
3110 "* [Northern Ireland Statistics and Research Agency](https://www.nisra.gov.uk/publications/weekly-deaths) (Northern Ireland). Weeks start on a Saturday. Note that the week numbers don't match the England and Wales data.\n",
3111 "* [National Records of Scotland](https://www.nrscotland.gov.uk/statistics-and-data/statistics/statistics-by-theme/vital-events/general-publications/weekly-and-monthly-data-on-births-and-deaths/weekly-data-on-births-and-deaths) (Scotland). Note that Scotland uses ISO8601 week numbers, which start on a Monday.\"\"\")\n",
3112 " \n",
3113 " f.write('\\n\\n')\n",
3114 " f.write('> [Source code available](https://git.njae.me.uk/?p=covid19.git;a=tree)\\n')\n",
3115 " f.write('\\n') \n"
3116 ]
3117 },
3118 {
3119 "cell_type": "code",
3120 "execution_count": null,
3121 "metadata": {
3122 "Collapsed": "false"
3123 },
3124 "outputs": [],
3125 "source": [
3126 "!pandoc --toc -s covid_summary.md > covid_summary.html"
3127 ]
3128 },
3129 {
3130 "cell_type": "code",
3131 "execution_count": null,
3132 "metadata": {
3133 "Collapsed": "false"
3134 },
3135 "outputs": [],
3136 "source": [
3137 "!scp covid_summary.html neil@ogedei:/var/www/scripts.njae.me.uk/covid/index.html\n",
3138 "!scp covid_deaths_total_linear.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3139 "!scp deaths-radar.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3140 "!scp covid_deaths_per_day_7.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3141 "!scp covid_doubling_times_7.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3142 "!scp cases_per_day_with_lockdown.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3143 "!scp cases_admissions_deaths.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3144 "!scp fraction_positive_tests.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/ \n",
3145 "!scp tests_and_cases.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3146 "!scp deaths_by_date_last_30_days.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3147 "!scp fraction_positive_tests_vs_tests.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3148 "!scp tests_vs_fraction_positive_animation.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/ "
3149 ]
3150 },
3151 {
3152 "cell_type": "code",
3153 "execution_count": null,
3154 "metadata": {
3155 "Collapsed": "false"
3156 },
3157 "outputs": [],
3158 "source": [
3159 "with open('uk_covid_deaths.js', 'w') as f:\n",
3160 " f.write(f\"document.write('{uk_covid_deaths}');\")\n",
3161 " \n",
3162 "with open('estimated_total_deaths.js', 'w') as f:\n",
3163 " f.write(f\"document.write('{uk_deaths_to_date:.0f}');\")\n",
3164 "\n",
3165 "# with open('projection_date.js', 'w') as f:\n",
3166 "# f.write(f\"document.write(\\'{projection_date.strftime('%d %B %Y')}\\');\")\n",
3167 "\n",
3168 "# with open('projected_deaths.js', 'w') as f:\n",
3169 "# f.write(f\"document.write('{uk_projection.deaths.sum():.0f}');\")\n",
3170 "\n",
3171 "# with open('projected_excess_deaths.js', 'w') as f:\n",
3172 "# f.write(f\"document.write('{deaths_actual_projected_scaled:.0f}');\")\n",
3173 "\n",
3174 "edut = pd.to_datetime(excess_deaths_data[\"end_date\"]).strftime('%d %B %Y')\n",
3175 "with open('excess_deaths_upto.js', 'w') as f:\n",
3176 " f.write(f\"document.write('{edut}');\")\n",
3177 "\n",
3178 "# with open('excess_deaths.js', 'w') as f:\n",
3179 "# f.write(f\"document.write('{excess_deaths:.0f}');\")\n",
3180 " \n",
3181 "# with open('reported_deaths.js', 'w') as f:\n",
3182 "# f.write(f\"document.write('{ons_reported_deaths:.0f}');\")\n",
3183 " \n",
3184 "# with open('scaling_factor.js', 'w') as f:\n",
3185 "# f.write(f\"document.write('{excess_death_accuracy:.2f}');\") \n",
3186 "\n",
3187 "# with open('projection_length.js', 'w') as f:\n",
3188 "# f.write(f\"document.write('{s_end - s_start - 1}');\")\n",
3189 " \n",
3190 "# with open('s_end.js', 'w') as f:\n",
3191 "# f.write(f\"document.write('{s_end}');\")\n",
3192 " \n",
3193 "# s_start_date_str = s_start_date.strftime(\"%d %B %Y\")\n",
3194 "# with open('s_start_date.js', 'w') as f:\n",
3195 "# f.write(f\"document.write('{s_start_date_str}');\")\n",
3196 " \n",
3197 "# with open('uk_end.js', 'w') as f:\n",
3198 "# f.write(f\"document.write('{uk_end}');\")\n",
3199 " \n",
3200 "with open('last_uk_date.js', 'w') as f:\n",
3201 " f.write(f\"document.write('{pd.to_datetime(last_uk_date).strftime('%d %B %Y')}');\")"
3202 ]
3203 },
3204 {
3205 "cell_type": "code",
3206 "execution_count": null,
3207 "metadata": {
3208 "Collapsed": "false"
3209 },
3210 "outputs": [],
3211 "source": [
3212 "# pd.to_datetime(excess_deaths_upto).strftime('%d %B %Y')"
3213 ]
3214 },
3215 {
3216 "cell_type": "code",
3217 "execution_count": null,
3218 "metadata": {
3219 "Collapsed": "false"
3220 },
3221 "outputs": [],
3222 "source": [
3223 "!scp uk_covid_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3224 "!scp estimated_total_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3225 "# !scp projection_date.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3226 "# !scp projected_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3227 "# !scp projected_excess_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3228 "!scp excess_deaths_upto.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3229 "# !scp excess_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3230 "# !scp reported_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3231 "# !scp scaling_factor.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3232 "# !scp projection_length.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3233 "# !scp s_end.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3234 "# !scp s_start_date.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3235 "# !scp uk_end.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3236 "!scp last_uk_date.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/\n",
3237 "!scp hospital_normalisation_date.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/"
3238 ]
3239 },
3240 {
3241 "cell_type": "code",
3242 "execution_count": null,
3243 "metadata": {
3244 "Collapsed": "false"
3245 },
3246 "outputs": [],
3247 "source": [
3248 "data_by_date.loc['UK'].to_csv('data_by_day_uk.csv', header=True, index=True)\n",
3249 "data_by_date.loc['BE'].to_csv('data_by_day_be.csv', header=True, index=True)"
3250 ]
3251 },
3252 {
3253 "cell_type": "code",
3254 "execution_count": null,
3255 "metadata": {
3256 "Collapsed": "false"
3257 },
3258 "outputs": [],
3259 "source": [
3260 "ukd = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), ['deaths', 'deaths_m7']].droplevel(1)\n",
3261 "ax = ukd.deaths.plot.bar(figsize=(12, 8))\n",
3262 "ukd.deaths_m7.plot.line(ax=ax, color='red')\n",
3263 "# ax = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), 'deaths_m7'].plot.line(figsize=(12, 8), color='red')\n",
3264 "# ax = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), 'deaths'].plot.bar(ax=ax)\n",
3265 "ax.set_xticks(range(0, 120, 20))"
3266 ]
3267 },
3268 {
3269 "cell_type": "code",
3270 "execution_count": null,
3271 "metadata": {
3272 "Collapsed": "false"
3273 },
3274 "outputs": [],
3275 "source": [
3276 "ukdd = data_by_date.loc['UK'].iloc[-30:]\n",
3277 "ax = ukdd.deaths_m7.plot.line(figsize=(12, 8), color='red')\n",
3278 "# ukdd.deaths.plot.bar(ax=ax)\n",
3279 "ax.bar(ukdd.index, ukdd.deaths)"
3280 ]
3281 },
3282 {
3283 "cell_type": "code",
3284 "execution_count": null,
3285 "metadata": {
3286 "Collapsed": "false"
3287 },
3288 "outputs": [],
3289 "source": [
3290 "ukdd"
3291 ]
3292 },
3293 {
3294 "cell_type": "code",
3295 "execution_count": null,
3296 "metadata": {
3297 "Collapsed": "false"
3298 },
3299 "outputs": [],
3300 "source": [
3301 "np.arange(0, 130, 20)"
3302 ]
3303 },
3304 {
3305 "cell_type": "code",
3306 "execution_count": null,
3307 "metadata": {
3308 "Collapsed": "false"
3309 },
3310 "outputs": [],
3311 "source": [
3312 "data_by_date.loc['UK']"
3313 ]
3314 },
3315 {
3316 "cell_type": "code",
3317 "execution_count": null,
3318 "metadata": {
3319 "Collapsed": "false"
3320 },
3321 "outputs": [],
3322 "source": [
3323 "data_by_date.loc['UK'].plot(x='deaths_culm', y='deaths', logx=True, logy=True)"
3324 ]
3325 },
3326 {
3327 "cell_type": "code",
3328 "execution_count": null,
3329 "metadata": {
3330 "Collapsed": "false"
3331 },
3332 "outputs": [],
3333 "source": [
3334 "data_by_date.loc['UK'].plot(x='cases_culm', y='cases')"
3335 ]
3336 },
3337 {
3338 "cell_type": "code",
3339 "execution_count": null,
3340 "metadata": {
3341 "Collapsed": "false"
3342 },
3343 "outputs": [],
3344 "source": [
3345 "ukdbd = data_by_date.loc['UK'].copy()\n",
3346 "ukdbd['deaths_m7'] = ukdbd.deaths.transform(lambda x: x.rolling(7, 1).mean())\n",
3347 "ukdbd['cases_m7'] = ukdbd.cases.transform(lambda x: x.rolling(7, 1).mean())\n",
3348 "ukdbd"
3349 ]
3350 },
3351 {
3352 "cell_type": "code",
3353 "execution_count": null,
3354 "metadata": {
3355 "Collapsed": "false"
3356 },
3357 "outputs": [],
3358 "source": [
3359 "ukdbd.plot(x='deaths_culm', y='deaths_m7', logx=True, logy=True)"
3360 ]
3361 },
3362 {
3363 "cell_type": "code",
3364 "execution_count": null,
3365 "metadata": {
3366 "Collapsed": "false"
3367 },
3368 "outputs": [],
3369 "source": [
3370 "fig, ax = plt.subplots(figsize=(12, 8))\n",
3371 "xmax = 10\n",
3372 "for c in COUNTRIES_CORE:\n",
3373 " if data_since_threshold.loc[(slice(None), c), 'deaths_culm'].max() > xmax:\n",
3374 " xmax = data_since_threshold.loc[(slice(None), c), 'deaths_culm'].max()\n",
3375 " data_since_threshold.loc[(slice(None), c), :].plot(x='deaths_culm', y='deaths_m7', logx=True, logy=True, xlim=(10, xmax * 1.1), label=c, ax=ax)"
3376 ]
3377 },
3378 {
3379 "cell_type": "code",
3380 "execution_count": null,
3381 "metadata": {
3382 "Collapsed": "false"
3383 },
3384 "outputs": [],
3385 "source": [
3386 "data_since_threshold.loc[(slice(None), 'UK'), 'deaths_culm'].max()"
3387 ]
3388 },
3389 {
3390 "cell_type": "code",
3391 "execution_count": null,
3392 "metadata": {
3393 "Collapsed": "false"
3394 },
3395 "outputs": [],
3396 "source": [
3397 "countries.continentExp.unique()"
3398 ]
3399 },
3400 {
3401 "cell_type": "code",
3402 "execution_count": null,
3403 "metadata": {
3404 "Collapsed": "false"
3405 },
3406 "outputs": [],
3407 "source": [
3408 "countries.loc['KW']"
3409 ]
3410 },
3411 {
3412 "cell_type": "code",
3413 "execution_count": null,
3414 "metadata": {
3415 "Collapsed": "false"
3416 },
3417 "outputs": [],
3418 "source": [
3419 "data_by_date.groupby(level=0)['deaths'].shift(-25)"
3420 ]
3421 },
3422 {
3423 "cell_type": "code",
3424 "execution_count": null,
3425 "metadata": {
3426 "Collapsed": "false"
3427 },
3428 "outputs": [],
3429 "source": [
3430 "offset_data = data_by_date.loc[:, ['cases']]\n",
3431 "offset_data['deaths'] = data_by_date.groupby(level=0)['deaths'].shift(-25)\n",
3432 "offset_data['cases_m7'] = offset_data.groupby(level=0)['cases'].transform(lambda x: x.rolling(7, 1).mean())\n",
3433 "offset_data['deaths_m7'] = offset_data['deaths'].dropna().groupby(level=0).transform(lambda x: x.rolling(7, 1).mean())\n",
3434 "offset_data['deaths_per_case'] = offset_data.deaths_m7 / offset_data.cases_m7\n",
3435 "offset_data"
3436 ]
3437 },
3438 {
3439 "cell_type": "code",
3440 "execution_count": null,
3441 "metadata": {
3442 "Collapsed": "false"
3443 },
3444 "outputs": [],
3445 "source": [
3446 "deaths_m7"
3447 ]
3448 },
3449 {
3450 "cell_type": "code",
3451 "execution_count": null,
3452 "metadata": {
3453 "Collapsed": "false"
3454 },
3455 "outputs": [],
3456 "source": [
3457 "offset_deaths_m7 = (offset_data.loc[COUNTRIES_ALL, ['deaths_m7']]\n",
3458 " .unstack().sort_index().xs('deaths_m7', axis=1, drop_level=True)).T.sort_index()\n",
3459 "offset_deaths_m7"
3460 ]
3461 },
3462 {
3463 "cell_type": "code",
3464 "execution_count": null,
3465 "metadata": {
3466 "Collapsed": "false"
3467 },
3468 "outputs": [],
3469 "source": [
3470 "offset_deaths_m7['UK']"
3471 ]
3472 },
3473 {
3474 "cell_type": "code",
3475 "execution_count": null,
3476 "metadata": {
3477 "Collapsed": "false"
3478 },
3479 "outputs": [],
3480 "source": [
3481 "data_since_threshold.loc[(slice(None), 'UK'), :].tail()"
3482 ]
3483 },
3484 {
3485 "cell_type": "code",
3486 "execution_count": null,
3487 "metadata": {
3488 "Collapsed": "false"
3489 },
3490 "outputs": [],
3491 "source": [
3492 "countries.loc['PT']"
3493 ]
3494 },
3495 {
3496 "cell_type": "code",
3497 "execution_count": null,
3498 "metadata": {
3499 "Collapsed": "false"
3500 },
3501 "outputs": [],
3502 "source": [
3503 "ax = cases_by_date_m7.iloc[-50:][COUNTRIES_FRIENDS].plot(figsize=(15, 9), title=\"Cases per day, 7 day moving average\")#, ylim=(-10, 1500))\n",
3504 "# ax.set_xlabel(f\"Days since {DEATH_COUNT_THRESHOLD} deaths\")\n",
3505 "# uk_projection.deaths_m7.plot(ax=ax)\n",
3506 "for c in COUNTRIES_FRIENDS:\n",
3507 " lvi = cases_by_date_m7[c].last_valid_index()\n",
3508 " ax.text(x = lvi + pd.Timedelta(days=1), y = cases_by_date_m7[c][lvi], s = f\"{c}: {cases_by_date_m7[c][lvi]:.0f}\")\n"
3509 ]
3510 },
3511 {
3512 "cell_type": "code",
3513 "execution_count": null,
3514 "metadata": {
3515 "Collapsed": "false"
3516 },
3517 "outputs": [],
3518 "source": [
3519 "ax = deaths_by_date_m7.iloc[-50:][COUNTRIES_FRIENDS].plot(figsize=(15, 9), title=\"Deaths per day, 7 day moving average\")#, ylim=(-10, 100))\n",
3520 "# ax.set_xlabel(f\"Days since {DEATH_COUNT_THRESHOLD} deaths\")\n",
3521 "# uk_projection.deaths_m7.plot(ax=ax)\n",
3522 "for c in COUNTRIES_FRIENDS:\n",
3523 " lvi = deaths_by_date_m7[c].last_valid_index()\n",
3524 "# if c != 'ES':\n",
3525 " ax.text(x = lvi + pd.Timedelta(days=1), y = deaths_by_date_m7[c][lvi], s = f\"{c}: {deaths_by_date_m7[c][lvi]:.0f}\")"
3526 ]
3527 },
3528 {
3529 "cell_type": "code",
3530 "execution_count": null,
3531 "metadata": {
3532 "Collapsed": "false"
3533 },
3534 "outputs": [],
3535 "source": []
3536 }
3537 ],
3538 "metadata": {
3539 "jupytext": {
3540 "formats": "ipynb,md"
3541 },
3542 "kernelspec": {
3543 "display_name": "Python 3",
3544 "language": "python",
3545 "name": "python3"
3546 },
3547 "language_info": {
3548 "codemirror_mode": {
3549 "name": "ipython",
3550 "version": 3
3551 },
3552 "file_extension": ".py",
3553 "mimetype": "text/x-python",
3554 "name": "python",
3555 "nbconvert_exporter": "python",
3556 "pygments_lexer": "ipython3",
3557 "version": "3.7.4"
3558 }
3559 },
3560 "nbformat": 4,
3561 "nbformat_minor": 4
3562 }
Code for analysing Covid-19 data