Collecting various updates
[covid19.git] / covid.md
index 5b762de0275f32355e221821b654967a178b797e..e86f0ed5e62e5037ca833da38741694937b12bcd 100644 (file)
--- a/covid.md
+++ b/covid.md
@@ -18,9 +18,11 @@ Data from [European Centre for Disease Prevention and Control](https://www.ecdc.
 ```python
 import itertools
 import collections
+import json
 import pandas as pd
 import numpy as np
 from scipy.stats import gmean
+import datetime
 
 import matplotlib as mpl
 import matplotlib.pyplot as plt
@@ -31,9 +33,15 @@ import matplotlib.pyplot as plt
 DEATH_COUNT_THRESHOLD = 10
 COUNTRIES_CORE = 'IT DE UK ES IE FR'.split()
 COUNTRIES_NORDIC = 'SE NO DK FI UK'.split()
-COUNTRIES_FRIENDS = 'IT UK ES BE SI MX'.split()
+# COUNTRIES_FRIENDS = 'IT UK ES BE SI MX'.split()
+COUNTRIES_FRIENDS = 'IT UK ES BE SI PT'.split()
+
+COUNTRIES_AMERICAS = ['AG', 'AR', 'AW', 'BS', 'BB', 'BZ', 'BM', 'BO', 'BR', 'VG', 'KY', # excluding Canada and USA
+       'CL', 'CO', 'CR', 'CU', 'CW', 'DM', 'DO', 'EC', 'SV', 'GL', 'GD', 'GT',
+       'GY', 'HT', 'HN', 'JM', 'MX', 'MS', 'NI', 'PA', 'PY', 'PE', 'PR', 'KN',
+       'LC', 'VC', 'SX', 'SR', 'TT', 'TC', 'VI', 'UY', 'VE']
 COUNTRIES_OF_INTEREST = list(set(COUNTRIES_CORE + COUNTRIES_FRIENDS))
-COUNTRIES_ALL = list(set(COUNTRIES_CORE + COUNTRIES_FRIENDS + COUNTRIES_NORDIC))
+COUNTRIES_ALL = list(set(COUNTRIES_CORE + COUNTRIES_FRIENDS + COUNTRIES_NORDIC + COUNTRIES_AMERICAS))
 ```
 
 ```python
@@ -42,13 +50,31 @@ COUNTRIES_ALL = list(set(COUNTRIES_CORE + COUNTRIES_FRIENDS + COUNTRIES_NORDIC))
 
 ```python
 # First col is a date, treat geoId of NA as 'Namibia', not "NA" value
-raw_data = pd.read_csv('covid.csv', parse_dates=[0], keep_default_na=False, dayfirst=True)
+raw_data = pd.read_csv('covid.csv', 
+                       parse_dates=[0], dayfirst=True,
+                       keep_default_na=False, na_values = [''],
+#                        dtype = {'day': np.int64, 
+#                                 'month': np.int64, 
+#                                 'year': np.int64, 
+#                                 'cases': np.int64, 
+#                                 'deaths': np.int64, 
+#                                 'countriesAndTerritories': str, 
+#                                 'geoId': str, 
+#                                 'countryterritoryCode': str, 
+#                                 'popData2019': np.int64, 
+#                                 'continentExp': str, 
+#                                 }
+                      )
 ```
 
 ```python
 raw_data.size
 ```
 
+```python
+raw_data.fillna(0, inplace=True)
+```
+
 ```python
 raw_data.head()
 ```
@@ -58,25 +84,47 @@ raw_data.dtypes
 ```
 
 ```python
-base_data = raw_data.set_index(['geoId', 'dateRep'])
-base_data.sort_index(inplace=True)
-base_data
+raw_data = raw_data.astype({'dateRep': np.datetime64, 
+    'day': np.int64, 
+    'month': np.int64, 
+    'year': np.int64, 
+    'cases': np.int64, 
+    'deaths': np.int64, 
+    'countriesAndTerritories': str, 
+    'geoId': str, 
+    'countryterritoryCode': str, 
+    'popData2019': np.int64, 
+    'continentExp': str })
+```
+
+```python
+raw_data.dtypes
+```
+
+```python
+raw_data[((raw_data.geoId == 'UK') & (raw_data.dateRep >= '2020-07-10'))]
 ```
 
 ```python
-base_data.loc['UK']
+# raw_data = raw_data[~ ((raw_data.geoId == 'ES') & (raw_data.dateRep >= '2020-05-22'))]
 ```
 
 ```python
-base_data.loc['UK', '2020-04-17']
+base_data = raw_data.set_index(['geoId', 'dateRep'])
+base_data.sort_index(inplace=True)
+base_data
 ```
 
 ```python
-countries = raw_data[['geoId', 'countriesAndTerritories', 'popData2018']]
-countries = countries[countries['popData2018'] != '']
+base_data.loc['ES'].loc['2020-05-10':]
+```
+
+```python
+countries = raw_data[['geoId', 'countriesAndTerritories', 'popData2019', 'continentExp']]
+countries = countries[countries['popData2019'] != '']
 countries = countries.drop_duplicates()
 countries.set_index('geoId', inplace=True)
-countries = countries.astype({'popData2018': 'int64'})
+countries = countries.astype({'popData2019': 'int64'})
 countries.head()
 ```
 
@@ -92,6 +140,10 @@ countries[countries.countriesAndTerritories == 'Finland']
 countries.loc[COUNTRIES_OF_INTEREST]
 ```
 
+```python
+countries[countries.continentExp == 'America'].index
+```
+
 ```python
 data_by_date = base_data[['cases', 'deaths']]
 data_by_date.head()
@@ -101,6 +153,10 @@ data_by_date.head()
 data_by_date.loc['UK']
 ```
 
+```python
+# data_by_date.deaths.drop_duplicates().sort_values().to_csv('dth.csv', header=True)
+```
+
 ```python
 data_by_date.groupby(level=0).cumsum()
 ```
@@ -170,6 +226,7 @@ data_since_threshold
 data_since_threshold = data_since_threshold.set_index('since_threshold', append=True
                               ).reorder_levels(['since_threshold', 'geoId', 'dateRep']
                                               ).reset_index('dateRep')
+data_since_threshold.sort_index(inplace=True)
 data_since_threshold
 ```
 
@@ -177,6 +234,10 @@ data_since_threshold
 data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), :]
 ```
 
+```python
+data_since_threshold.loc[(slice(None), ['ES']), :].tail(8)
+```
+
 ```python
 data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), ['deaths_culm']].unstack().plot(logy=True)
 ```
@@ -189,6 +250,14 @@ data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), ['deaths_culm']].uns
 deaths = data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['deaths_culm']].unstack().sort_index().xs('deaths_culm', axis=1, drop_level=True)
 ```
 
+```python
+cases = data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['cases_culm']].unstack().sort_index().xs('cases_culm', axis=1, drop_level=True)
+```
+
+```python
+COUNTRIES_AMERICAS_DEAD = list(set(deaths.columns) & set(COUNTRIES_AMERICAS))
+```
+
 ```python
 data_since_threshold.reset_index().merge(countries, on='geoId').set_index(['since_threshold', 'geoId'])
 ```
@@ -199,8 +268,8 @@ data_since_threshold.reset_index().merge(countries, on='geoId').set_index(['sinc
 
 ```python
 data_since_threshold_per_capita = data_since_threshold.reset_index().merge(countries, on='geoId').set_index(['since_threshold', 'geoId'])
-data_since_threshold_per_capita['cases_culm_pc'] = data_since_threshold_per_capita.cases_culm / data_since_threshold_per_capita.popData2018
-data_since_threshold_per_capita['deaths_culm_pc'] = data_since_threshold_per_capita.deaths_culm / data_since_threshold_per_capita.popData2018
+data_since_threshold_per_capita['cases_culm_pc'] = data_since_threshold_per_capita.cases_culm / data_since_threshold_per_capita.popData2019
+data_since_threshold_per_capita['deaths_culm_pc'] = data_since_threshold_per_capita.deaths_culm / data_since_threshold_per_capita.popData2019
 data_since_threshold_per_capita
 ```
 
@@ -208,10 +277,6 @@ data_since_threshold_per_capita
 deaths_pc = data_since_threshold_per_capita.loc[(slice(None), ['UK', 'DE', 'IT', 'IE']), ['deaths_culm_pc']].unstack().sort_index().xs('deaths_culm_pc', axis=1, drop_level=True)
 ```
 
-```python
-deaths_pc
-```
-
 ```python
 deaths_pc.index
 ```
@@ -228,9 +293,57 @@ deaths[COUNTRIES_CORE].plot()
 deaths[COUNTRIES_FRIENDS].plot()
 ```
 
+```python
+ax = deaths[COUNTRIES_FRIENDS].plot(figsize=(10, 6), title="Total deaths, linear")
+ax.set_xlabel(f"Days since {DEATH_COUNT_THRESHOLD} deaths")
+for c in COUNTRIES_FRIENDS:
+    lvi = deaths[c].last_valid_index()
+    ax.text(x = lvi + 1, y = deaths[c][lvi], s = f"{c}: {deaths[c][lvi]:.0f}")
+# plt.savefig('covid_deaths_total_linear.png')    
+```
+
 ```python
 ax = deaths[COUNTRIES_CORE].plot(figsize=(10, 6), title="Total deaths, linear")
+ax.set_xlabel(f"Days since {DEATH_COUNT_THRESHOLD} deaths")
 for c in COUNTRIES_CORE:
+    lvi = deaths[c].last_valid_index()
+    ax.text(x = lvi + 1, y = deaths[c][lvi], s = f"{c}: {deaths[c][lvi]:.0f}")
+plt.savefig('covid_deaths_total_linear.png')    
+```
+
+```python
+deaths_prime = deaths[COUNTRIES_CORE].copy()
+deaths_prime.loc[73:, 'ES'] = np.NaN
+# deaths_prime['ES'][70:]
+```
+
+```python
+ax = deaths_prime[COUNTRIES_CORE].plot(figsize=(10, 6), title="Total deaths, linear")
+for c in COUNTRIES_CORE:
+    lvi = deaths_prime[c].last_valid_index()
+    ax.text(x = lvi + 1, y = deaths_prime[c][lvi], s = f"{c}: {deaths_prime[c][lvi]:.0f}")
+# plt.savefig('covid_deaths_total_linear.png')    
+```
+
+```python
+ax = cases[COUNTRIES_CORE].plot(figsize=(10, 6), title="Total cases, linear")
+for c in COUNTRIES_CORE:
+    lvi = cases[c].last_valid_index()
+    ax.text(x = lvi + 1, y = cases[c][lvi], s = c)
+plt.savefig('covid_cases_total_linear.png')    
+```
+
+```python
+ax = deaths[COUNTRIES_AMERICAS_DEAD].plot(figsize=(10, 6), title="Total deaths, linear")
+for c in COUNTRIES_AMERICAS_DEAD:
+    lvi = deaths[c].last_valid_index()
+    ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)
+# plt.savefig('covid_deaths_total_linear.png')    
+```
+
+```python
+ax = deaths[COUNTRIES_CORE + ['BR', 'MX']].plot(figsize=(10, 6), title="Total deaths, linear")
+for c in COUNTRIES_CORE + ['BR', 'MX']:
     lvi = deaths[c].last_valid_index()
     ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)
 # plt.savefig('covid_deaths_total_linear.png')    
@@ -249,7 +362,7 @@ ax = deaths[COUNTRIES_OF_INTEREST].plot(figsize=(10, 6), title="Total deaths, li
 for c in COUNTRIES_OF_INTEREST:
     lvi = deaths[c].last_valid_index()
     ax.text(x = lvi + 1, y = deaths[c][lvi], s = c)
-plt.savefig('covid_deaths_total_linear.png') 
+plt.savefig('covid_deaths_total_linear_of_interest.png') 
 ```
 
 ```python
@@ -300,7 +413,10 @@ plt.savefig('covid_deaths_total_log.png')
 ```
 
 ```python
-deaths_pc.plot(figsize=(10, 6), title="Deaths per capita, linear")
+ax = deaths_pc.plot(figsize=(10, 6), title="Deaths per capita, linear")
+for c in deaths_pc.columns:
+    lvi = deaths_pc[c].last_valid_index()
+    ax.text(x = lvi + 1, y = deaths_pc[c][lvi], s = c)
 plt.savefig('covid_deaths_per_capita_linear.png')
 ```
 
@@ -335,6 +451,7 @@ data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), :]
 ```python
 data_since_threshold['deaths_m4'] = data_since_threshold.groupby(level=1)['deaths'].transform(lambda x: x.rolling(4, 1).mean())
 data_since_threshold['deaths_m7'] = data_since_threshold.groupby(level=1)['deaths'].transform(lambda x: x.rolling(7, 1).mean())
+data_since_threshold['cases_m7'] = data_since_threshold.groupby(level=1)['cases'].transform(lambda x: x.rolling(7, 1).mean())
 # data_since_threshold['deaths_diff_m4'] = data_since_threshold.groupby(level=1)['deaths_diff'].transform(lambda x: x.rolling(4, 1).mean())
 # data_since_threshold['deaths_diff_m7'] = data_since_threshold.groupby(level=1)['deaths_diff'].transform(lambda x: x.rolling(7, 1).mean())
 data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), :]
@@ -350,6 +467,11 @@ deaths_m7 = (data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['deaths_m7'
              .unstack().sort_index().xs('deaths_m7', axis=1, drop_level=True))
 ```
 
+```python
+cases_m7 = (data_since_threshold.loc[(slice(None), COUNTRIES_ALL), ['cases_m7']]
+             .unstack().sort_index().xs('cases_m7', axis=1, drop_level=True))
+```
+
 ```python
 ax = deaths_m4.plot(figsize=(10, 6), title="Deaths per day, 4 day moving average")
 for c in deaths_m4.columns:
@@ -385,12 +507,38 @@ for c in C7s:
 
 ```python
 ax = deaths_m7[COUNTRIES_CORE].plot(figsize=(10, 6), title="Deaths per day, 7 day moving average")
+ax.set_xlabel(f"Days since {DEATH_COUNT_THRESHOLD} deaths")
 for c in COUNTRIES_CORE:
     lvi = deaths_m7[c].last_valid_index()
     ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = c)
+# plt.axhline(0, color='0.7')
 plt.savefig('covid_deaths_per_day_7.png') 
 ```
 
+```python
+ax = deaths_m7[COUNTRIES_FRIENDS].plot(figsize=(10, 6), title="Deaths per day, 7 day moving average")
+ax.set_xlabel(f"Days since {DEATH_COUNT_THRESHOLD} deaths")
+for c in COUNTRIES_FRIENDS:
+    lvi = deaths_m7[c].last_valid_index()
+    ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = c)
+# plt.axhline(0, color='0.7')
+# plt.savefig('covid_deaths_per_day_7.png') 
+```
+
+```python
+deaths_m7_prime = deaths_m7[COUNTRIES_CORE].copy()
+deaths_m7_prime.loc[73:, 'ES'] = np.NaN
+deaths_m7_prime['ES'][70:]
+```
+
+```python
+ax = deaths_m7_prime[COUNTRIES_CORE].plot(figsize=(10, 6), title="Deaths per day, 7 day moving average")
+for c in COUNTRIES_CORE:
+    lvi = deaths_m7_prime[c].last_valid_index()
+    ax.text(x = lvi + 1, y = deaths_m7_prime[c][lvi], s = c)
+# plt.savefig('covid_deaths_per_day_7.png') # see below for where this is written, with the projection
+```
+
 ```python
 ax = deaths_m7[COUNTRIES_FRIENDS].plot(figsize=(10, 6), title="Deaths per day, 7 day moving average")
 for c in COUNTRIES_FRIENDS:
@@ -399,6 +547,30 @@ for c in COUNTRIES_FRIENDS:
 plt.savefig('covid_deaths_per_day_friends_7.png') 
 ```
 
+```python
+ax = deaths_m7[COUNTRIES_CORE + ['BR', 'MX']].plot(figsize=(10, 6), title="Deaths per day, 7 day moving average")
+for c in COUNTRIES_CORE + ['BR', 'MX']:
+    lvi = deaths_m7[c].last_valid_index()
+    ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = c)
+# plt.savefig('covid_deaths_per_day_7.png') 
+```
+
+```python
+ax = cases_m7[COUNTRIES_CORE].plot(figsize=(10, 6), title="Cases per day, 7 day moving average")
+for c in COUNTRIES_CORE:
+    lvi = cases_m7[c].last_valid_index()
+    ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = c)
+plt.savefig('covid_cases_per_day-core.png') 
+```
+
+```python
+ax = cases_m7[COUNTRIES_FRIENDS].plot(figsize=(10, 6), title="Cases per day, 7 day moving average")
+for c in COUNTRIES_FRIENDS:
+    lvi = cases_m7[c].last_valid_index()
+    ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = c)
+# plt.savefig('covid_cases_per_day-core.png') 
+```
+
 ```python
 def gmean_scale(items):
     return gmean(items) / items[-1]
@@ -424,8 +596,8 @@ data_since_threshold.loc[(slice(None), ['UK', 'DE', 'IT']), :]
 ```
 
 ```python
-data_since_threshold['doubling_time'] = data_since_threshold.groupby(level=1).apply(doubling_time).reset_index(level=0, drop=True)
-data_since_threshold['doubling_time_7'] = data_since_threshold.groupby(level=1).apply(doubling_time_7).reset_index(level=0, drop=True)
+data_since_threshold['doubling_time'] = data_since_threshold.groupby(level=1).apply(doubling_time).reset_index(level=0, drop=True).sort_index()
+data_since_threshold['doubling_time_7'] = data_since_threshold.groupby(level=1).apply(doubling_time_7).reset_index(level=0, drop=True).sort_index()
 # data_since_threshold.loc[(slice(None), 'UK'), :]
 ```
 
@@ -451,6 +623,7 @@ for c in doubling_times.columns:
 
 ```python
 ax = doubling_times_7[COUNTRIES_CORE].plot(figsize=(10, 6), title="Doubling times, 7 day average")
+ax.legend(loc="upper left")
 for c in COUNTRIES_CORE:
     lvi = doubling_times_7[c].last_valid_index()
     ax.text(x = lvi + 1, y = doubling_times_7[c][lvi], s = c)
@@ -496,7 +669,812 @@ for c in C7s:
 ```
 
 ```python
-data_since_threshold.replace([np.inf, -np.inf], np.nan).groupby(level=1).last().loc[COUNTRIES_ALL]#, [doubling_time]]
+# data_since_threshold.replace([np.inf, -np.inf], np.nan).groupby(level=1).last().loc[COUNTRIES_ALL]#, [doubling_time]]
+```
+
+```python
+dstl = data_since_threshold.replace([np.inf, -np.inf], np.nan).groupby(level=1).last()
+dstl.loc[dstl.index.intersection(COUNTRIES_ALL)]
+```
+
+```python
+# data_since_threshold.replace([np.inf, -np.inf], np.nan).groupby(level=1).last().loc[['UK', 'DE', 'IT']]#, [doubling_time]]
+dstl.loc[['UK', 'DE', 'IT', 'FR', 'ES']]
+```
+
+```python
+data_since_threshold.loc[(slice(None), ['UK']), :].tail(20)
+```
+
+```python
+data_since_threshold.loc[(slice(None), ['ES']), :].tail(20)
+```
+
+## Death projections
+
+```python
+data_since_threshold.loc[(slice(None), ['UK']), :].tail(15)
+```
+
+```python
+it_since_threshold = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['IT']), :]
+s_end = it_since_threshold.index.max()[0]
+s_end
+```
+
+```python
+uk_projection = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), :]
+uk_current_end = uk_projection.index.max()[0] + 1
+# s_start = uk_projection.index.max()[0] + 1
+uk_current_end
+```
+
+```python
+current_uk_deaths_m7 = uk_projection[uk_projection.deaths_m7 >= 0].iloc[-1].deaths_m7
+current_uk_deaths_m7
+```
+
+```python
+it_since_threshold[it_since_threshold.deaths_m7 <= current_uk_deaths_m7].loc[60:].first_valid_index()[0]
+```
+
+```python
+s_start = it_since_threshold[it_since_threshold.deaths_m7 <= current_uk_deaths_m7].loc[60:].first_valid_index()[0]
+s_start
+```
+
+```python
+s_start_date = data_since_threshold.loc[(89, 'IT'), 'dateRep']# .iloc[0]
+s_start_date
+```
+
+```python
+s_end - s_start
+```
+
+```python
+uk_end = s_end - s_start + uk_current_end
+uk_end
+```
+
+```python
+proj = it_since_threshold.loc[(slice(s_start, s_end), slice(None)), ['cases', 'deaths', 'deaths_m7']]
+ndiff = uk_current_end - s_start
+proj.index = pd.MultiIndex.from_tuples([(n + ndiff, 'UK') for n, _ in proj.index], names=proj.index.names)
+proj
+```
+
+```python
+it_since_threshold.loc[(slice(s_start - 8, s_start + 2), slice(None)), ['cases', 'deaths', 'deaths_m7']]
+```
+
+```python
+uk_projection[['cases', 'deaths', 'deaths_m7']].tail()
+```
+
+```python
+# proj['deaths_m7'] = proj['deaths_m7'] + 20
+# proj
+```
+
+Projected deaths, UK following IT trend from now.
+
+```python
+uk_projection = uk_projection.append(proj, sort=True)
+uk_projection.deaths.sum()
+```
+
+```python
+uk_projection = uk_projection.droplevel(1)
+uk_projection
+```
+
+```python
+uk_projection.loc[152, 'deaths']
+```
+
+## Correction for cumulative deaths correction on 14 August
+
+```python
+# uk_projection.loc[152, 'deaths'] = 50
+```
+
+```python
+uk_projection['deaths_m7'] = uk_projection['deaths'].transform(lambda x: x.rolling(7, 1).mean())
+uk_projection.loc[(uk_current_end - 20):(uk_current_end + 5)]
+```
+
+```python
+uk_projection.loc[(uk_current_end - 5):]
+```
+
+```python
+uk_projection.deaths_m7.plot()
+```
+
+```python
+proj.droplevel(level=1)
+```
+
+```python
+ax = deaths_m7[COUNTRIES_CORE].plot()
+# uk_projection['deaths_m7'].plot(figsize=(10, 6), title="Deaths per day, 7 day moving average", label="Projection", style='--', ax=ax)
+proj.droplevel(level=1)['deaths_m7'].plot(figsize=(10, 6), title="Deaths per day, 7 day moving average", label="Projection", style='--', ax=ax)
+ax.set_xlabel(f"Days since {DEATH_COUNT_THRESHOLD} deaths")
+for c in COUNTRIES_CORE:
+    lvi = deaths_m7[c].last_valid_index()
+    ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = c)
+plt.savefig('covid_deaths_per_day_7.png') 
+```
+
+```python
+it_since_threshold.deaths.sum()
+```
+
+# Excess deaths calculation
+
+```python
+with open('excess_deaths.json') as f:
+    excess_deaths_data = json.load(f)
+    
+with open('excess_death_accuracy.json') as f:
+    excess_death_accuracy = json.load(f)
+    
+excess_deaths_data, excess_death_accuracy
+```
+
+```python
+# excess_deaths_upto = '2020-05-08'
+# excess_deaths = 54500
+```
+
+```python
+excess_deaths_upto = excess_deaths_data['end_date']
+excess_deaths = excess_deaths_data['excess_deaths']
+```
+
+Recorded deaths in period where ONS has reported total deaths
+
+```python
+ons_reported_deaths = base_data.loc['UK'][:excess_deaths_upto]['deaths'].sum()
+ons_reported_deaths
+```
+
+```python
+excess_deaths_upto
+```
+
+## Correction for deaths total correction on 14 August
+
+```python
+ons_unreported_deaths_data = base_data.loc['UK'][excess_deaths_upto:].iloc[1:]['deaths']
+# ons_unreported_deaths_data['2020-08-14'] = 50
+```
+
+```python
+ons_unreported_deaths = ons_unreported_deaths_data.sum()
+ons_unreported_deaths
+```
+
+```python
+scaled_ons_unreported_deaths = ons_unreported_deaths * excess_death_accuracy
+scaled_ons_unreported_deaths
+```
+
+```python
+uk_deaths_to_date = excess_deaths + scaled_ons_unreported_deaths
+uk_deaths_to_date
+```
+
+```python
+# data_since_threshold.loc[(slice(None), 'UK'), :][data_since_threshold.dateRep == excess_deaths_data['end_date']]
+```
+
+```python
+data_since_threshold[data_since_threshold.dateRep == excess_deaths_data['end_date']].loc[(slice(None), 'UK'), :]
+```
+
+```python
+ons_unreported_start = data_since_threshold[data_since_threshold.dateRep == excess_deaths_data['end_date']].loc[(slice(None), 'UK'), :].first_valid_index()[0] + 1
+ons_unreported_start
+```
+
+```python
+unreported_projected_deaths = uk_projection.loc[ons_unreported_start:].deaths.sum()
+unreported_projected_deaths
+```
+
+```python
+unreported_projected_deaths_scaled = unreported_projected_deaths * excess_death_accuracy
+unreported_projected_deaths_scaled
+```
+
+```python
+uk_projection.loc[(s_start):].deaths.sum()
+```
+
+```python
+deaths_actual_projected_scaled = uk_deaths_to_date + uk_projection.loc[(s_start):].deaths.sum() * excess_death_accuracy
+deaths_actual_projected_scaled
+```
+
+```python
+# excess_deaths / reported_deaths
+```
+
+True deaths to date, if we follow the scaling of excess deaths over reported deaths so far.
+
+```python
+uk_covid_deaths = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), 'deaths_culm'].iloc[-1]
+uk_covid_deaths
+```
+
+```python
+# uk_covid_deaths_scaled = excess_deaths + unreported_deaths * excess_death_accuracy
+# uk_covid_deaths_scaled
+```
+
+```python
+# data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['IT']), 'dateRep'].iloc[-1] + pd.Timedelta(s_end - s_start, unit='days')
+```
+
+```python
+# data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), 'dateRep'].iloc[-1].strftime("%Y-%m-%d")
+```
+
+```python
+# uk_covid_deaths * excess_deaths / reported_deaths
+```
+
+```python
+# uk_projection.deaths.sum() * excess_deaths / reported_deaths
+```
+
+```python
+# data_since_threshold.loc[(slice(None), 'FR'), :]
+# data_since_threshold[data_since_threshold.dateRep == '2020-05-18'].loc[(slice(None), 'FR'), :]
+```
+
+## School reopenings
+
+```python
+school_reopenings = {
+    'ES': {'date': '2020-05-18'},
+    'FR': {'date': '2020-05-18'}, # some areas only
+#     'IT': {'date': '2020-09-01'},
+    # 'IE': {'date': '2020-09-01'},
+    'DE': {'date': '2020-05-04'},
+    'UK': {'date': '2020-06-01'}
+}
+```
+
+```python
+data_since_threshold[data_since_threshold.dateRep == '2020-05-04'].loc[(slice(None), ['DE']), :].first_valid_index()
+```
+
+```python
+data_since_threshold[data_since_threshold.dateRep == '2020-05-04'].loc[(slice(None), ['DE']), :].iloc[0].deaths_m7
+```
+
+```python
+for cID in school_reopenings:
+    dst_in = data_since_threshold[data_since_threshold.dateRep == (school_reopenings[cID]['date'])].loc[(slice(None), [cID]), :]
+    dst_i = dst_in.first_valid_index()
+    dst_n = dst_in.iloc[0].deaths_m7
+    school_reopenings[cID]['since_threshold'] = dst_i[0]
+    school_reopenings[cID]['deaths_m7'] = dst_n
+school_reopenings
+```
+
+```python
+ax = deaths_m7[COUNTRIES_CORE].plot(figsize=(15, 9), title="Deaths per day, 7 day moving average")
+# uk_projection.deaths_m7.plot(ax=ax)
+for c in COUNTRIES_CORE:
+    lvi = deaths_m7[c].last_valid_index()
+    ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = f"{c}: {deaths_m7[c][lvi]:.0f}")
+    if c in school_reopenings:
+        marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()
+        ax.plot(school_reopenings[c]['since_threshold'], school_reopenings[c]['deaths_m7'], '*', 
+                markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)
+        ax.text(x = school_reopenings[c]['since_threshold'] + 1, y = school_reopenings[c]['deaths_m7'], 
+                s = f"{school_reopenings[c]['date']}: {school_reopenings[c]['deaths_m7']:.0f}")
+plt.savefig('school_reopenings.png')
+```
+
+```python
+# ax = deaths_m7[COUNTRIES_CORE].plot(figsize=(15, 9), title="Deaths per day, 7 day moving average",
+#                                    xlim=(46, 91), ylim=(0, 400))
+# # uk_projection.deaths_m7.plot(ax=ax)
+# for c in COUNTRIES_CORE:
+#     lvi = deaths_m7[c].last_valid_index()
+#     ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = f"{c}: {deaths_m7[c][lvi]:.0f}", fontsize=14)
+#     if c in school_reopenings:
+#         marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()
+#         ax.plot(school_reopenings[c]['since_threshold'], school_reopenings[c]['deaths_m7'], '*', 
+#                 markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)
+#         ax.text(x = school_reopenings[c]['since_threshold'] + 1, y = school_reopenings[c]['deaths_m7'], 
+#                 s = f"{school_reopenings[c]['date']}: {school_reopenings[c]['deaths_m7']:.0f}",
+#                 fontsize=14)
+# plt.savefig('school_reopenings_detail.png')
+```
+
+# Lockdown graphs
+
+```python
+lockdown_dates = {
+    'ES': { 'part_start': {'date': '2020-03-14'}
+          , 'full_start': {'date': '2020-03-15'}
+          , 'part_finish': {'date': '2020-05-18'}
+          },
+    'FR': { 'part_start': {'date': '2020-03-13'}
+          , 'full_start': {'date': '2020-03-17'}
+          , 'part_finish': {'date': '2020-05-11'}
+          },
+    'IT': { 'part_start': {'date': '2020-03-08'}
+          , 'full_start': {'date': '2020-03-10'}
+          , 'part_finish': {'date': '2020-05-04'}
+          },
+    'DE': { #'part_start': {'date': '2020-03-13'}
+          'full_start': {'date': '2020-03-22'}
+          , 'part_finish': {'date': '2020-05-06'}
+          },
+    'UK': { 'part_start': {'date': '2020-03-23'}
+          , 'full_start': {'date': '2020-03-23'}
+          , 'part_finish': {'date': '2020-05-31'}
+          },
+    'IE': { #'part_start': {'date': '2020-03-12'}
+          'full_start': {'date': '2020-03-27'}
+          , 'part_finish': {'date': '2020-05-18'}
+          },
+}
+```
+
+```python
+for cID in lockdown_dates:
+    for phase in lockdown_dates[cID]:
+        dst_in = data_since_threshold[data_since_threshold.dateRep == (lockdown_dates[cID][phase]['date'])].loc[(slice(None), [cID]), :]
+        dst_i = dst_in.first_valid_index()
+        dst_n = dst_in.iloc[0].deaths_m7
+        dst_c = dst_in.iloc[0].cases_m7
+        lockdown_dates[cID][phase]['since_threshold'] = dst_i[0]
+        lockdown_dates[cID][phase]['deaths_m7'] = dst_n
+        lockdown_dates[cID][phase]['cases_m7'] = dst_c
+
+lockdown_dates
+```
+
+```python
+ax = deaths_m7[COUNTRIES_CORE].plot(figsize=(15, 9), title="Deaths per day, 7 day moving averagee, with lockdown dates")
+ax.set_xlabel(f"Days since {DEATH_COUNT_THRESHOLD} deaths")
+# uk_projection.deaths_m7.plot(ax=ax)
+for c in COUNTRIES_CORE:
+    lvi = deaths_m7[c].last_valid_index()
+    if c != 'UK':
+        ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = f"{c}: {deaths_m7[c][lvi]:.0f}")
+    if c in lockdown_dates:
+        for phase in lockdown_dates[c]:
+            marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()
+            ax.plot(lockdown_dates[c][phase]['since_threshold'], lockdown_dates[c][phase]['deaths_m7'], '*',
+                    markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)
+            if 'start' not in phase:
+                ax.text(x = lockdown_dates[c][phase]['since_threshold'] + 1, y = lockdown_dates[c][phase]['deaths_m7'], 
+                        s = f"{lockdown_dates[c][phase]['date']}: {lockdown_dates[c][phase]['deaths_m7']:.0f}")
+# plt.savefig('school_reopenings.png')
+```
+
+```python
+ax = cases_m7.iloc[-50:][COUNTRIES_CORE].plot(figsize=(15, 9), title="Cases per day, 7 day moving average, with lockdown dates") #, ylim=(-10, 1500))
+ax.set_xlabel(f"Days since {DEATH_COUNT_THRESHOLD} deaths")
+# uk_projection.deaths_m7.plot(ax=ax)
+for c in COUNTRIES_CORE:
+    lvi = cases_m7[c].last_valid_index()
+#     if c != 'UK':
+    ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = f"{c}: {cases_m7[c][lvi]:.0f}")
+
+```
+
+```python
+ax = cases_m7[COUNTRIES_CORE].plot(figsize=(15, 9), title="Cases per day, 7 day moving average, with lockdown dates")
+ax.set_xlabel(f"Days since {DEATH_COUNT_THRESHOLD} deaths")
+# uk_projection.deaths_m7.plot(ax=ax)
+for c in COUNTRIES_CORE:
+    lvi = cases_m7[c].last_valid_index()
+#     if c != 'UK':
+    ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = f"{c}: {cases_m7[c][lvi]:.0f}")
+    if c in lockdown_dates:
+        for phase in lockdown_dates[c]:
+            marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()
+            if 'start' in phase:
+                marker_shape = '^'
+            else:
+                marker_shape = 'v'
+            ax.plot(lockdown_dates[c][phase]['since_threshold'], lockdown_dates[c][phase]['cases_m7'], 
+                    marker_shape,
+                    markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)
+            if 'start' not in phase:
+                ax.text(x = lockdown_dates[c][phase]['since_threshold'] + 1, y = lockdown_dates[c][phase]['cases_m7'], 
+                        s = f"{lockdown_dates[c][phase]['date']}: {lockdown_dates[c][phase]['cases_m7']:.0f}")
+plt.savefig('cases_per_day_with_lockdown.png')
+```
+
+```python
+ax = cases_m7[COUNTRIES_CORE].plot(figsize=(10, 6), title="Cases per day, 7 day moving average")
+for c in COUNTRIES_CORE:
+    lvi = cases_m7[c].last_valid_index()
+    ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = c)
+plt.savefig('covid_cases_per_day-core.png') 
+```
+
+```python
+ax = deaths_m7[COUNTRIES_CORE].plot(figsize=(15, 9), title="Deaths per day, 7 day moving average",
+                                   xlim=(0, 15), 
+                                    ylim=(0, 66)
+                                   )
+# uk_projection.deaths_m7.plot(ax=ax)
+for c in COUNTRIES_CORE:
+    lvi = deaths_m7[c].last_valid_index()
+    if c in lockdown_dates:
+        for phase in lockdown_dates[c]:
+            if 'start' in phase:
+                print(c, phase)
+                marker_col = [l for l in ax.lines if l.get_label() == c][0].get_color()
+                ax.plot(lockdown_dates[c][phase]['since_threshold'], lockdown_dates[c][phase]['deaths_m7'], '*', 
+                        markersize=18, markerfacecolor=marker_col, markeredgecolor=marker_col)
+                ax.text(x = lockdown_dates[c][phase]['since_threshold'] + 0.3, y = lockdown_dates[c][phase]['deaths_m7'], 
+                        s = f"{lockdown_dates[c][phase]['date']}: {lockdown_dates[c][phase]['deaths_m7']:.0f}")
+# plt.savefig('school_reopenings.png')
+```
+
+```python
+
+```
+
+```python
+
+```
+
+# Write results to summary file
+
+```python
+with open('covid_summary.md', 'w') as f:
+    f.write('% Covid death data summary\n')
+    f.write('% Neil Smith\n')
+    f.write(f'% Created on {datetime.datetime.now().strftime("%Y-%m-%d")}\n')
+    f.write('\n')
+        
+    last_uk_date = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), 'dateRep'].iloc[-1]
+    f.write(f'> Last UK data from {last_uk_date.strftime("%Y-%m-%d")}\n')
+```
+
+```python
+with open('covid_summary.md', 'a') as f:
+    f.write('\n')
+    f.write('## Headlines\n')
+    f.write('\n')
+    f.write('| []() | |\n')
+    f.write('|:---|---:|\n')
+    f.write(f'| Deaths reported so far | {uk_covid_deaths} | \n')
+    f.write(f'| Total Covid deaths to date (estimated) | {uk_deaths_to_date:.0f} |\n')
+    projection_date = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['IT']), 'dateRep'].iloc[-1] + pd.Timedelta(s_end - s_start, unit='days')
+    f.write(f'| Projected total deaths up to {projection_date.strftime("%Y-%m-%d")} | {deaths_actual_projected_scaled:.0f} | \n')
+    f.write('\n')
+```
+
+```python
+with open('covid_summary.md', 'a') as f:
+    f.write('\n')
+    f.write('## Total deaths\n')
+    f.write(f'Time based on days since {DEATH_COUNT_THRESHOLD} deaths\n')
+    f.write('\n')
+    f.write('![Total deaths](covid_deaths_total_linear.png)\n')
+    f.write('\n')
+    f.write('| Country ID | Country name | Total deaths |\n')
+    f.write('|:-----------|:-------------|-------------:|\n')
+    for c in sorted(COUNTRIES_CORE):
+        lvi = deaths[c].last_valid_index()
+        f.write(f'| {c} | {countries.loc[c].countriesAndTerritories} | {int(deaths[c][lvi])} |\n')
+    f.write('\n')
+```
+
+```python
+with open('covid_summary.md', 'a') as f:
+    f.write('\n')
+    f.write('## All-causes deaths, UK\n')
+    f.write('\n')
+    f.write('![All-causes deaths](deaths-radar.png)\n')
+    f.write('\n')
+    f.write('### Excess deaths\n')
+    f.write('\n')
+    f.write(f'From week ending 20 March 2020 until week ending {pd.to_datetime(excess_deaths_upto).strftime("%d %B %Y")}, ')
+    f.write(f'there were approximately **{excess_deaths:.0f}** excess deaths, over the average for the previous five years.\n')
+    f.write('\n')
+```
+
+```python
+with open('covid_summary.md', 'a') as f:
+    f.write(f'In that period, the UK reported {ons_reported_deaths} Covid deaths.\n')
+    f.write(f'In the last three weeks for which excess deaths have been reported, the excess deaths have been {excess_death_accuracy:.3f} higher than the Covid-reported deaths.\n')
+#     f.write(f'That means the actual number of Covid death is about {excess_deaths / reported_deaths:.2f} times higher than the reported figures.\n')
+    f.write('\n')
+    f.write(f'The UK has reported {uk_covid_deaths} deaths so far.\n')
+    f.write(f'Using the scaling factor above (for Covid-19 deaths after the ONS figures), I infer that there have been **{uk_deaths_to_date:.0f}** total deaths so far.\n')
+    f.write('\n')
+```
+
+```python
+with open('covid_summary.md', 'a') as f:
+    f.write('\n')
+    f.write('## Deaths per day\n')
+    f.write(f'Based on a 7-day moving average\n')
+    f.write('\n')
+    f.write('![Deaths per day](covid_deaths_per_day_7.png)\n')
+    f.write('\n')
+```
+
+```python
+s_end - s_start - 1
+```
+
+```python
+with open('covid_summary.md', 'a') as f:
+    f.write('\n')
+    f.write('## Projected deaths\n')
+    f.write(f"The UK's daily deaths data is very similar to Italy's.\n")
+    f.write(f'If I use the Italian data for the next {s_end - s_start - 1} days (from {s_start_date.strftime("%d %B %Y")} onwards),')
+    f.write(f' the UK will report {uk_projection.deaths.sum()} deaths on day {uk_end} of the epidemic.\n')
+    f.write('\n')
+    f.write('Using the excess deaths scaling from above, that will translate into ')
+    f.write(f'**{deaths_actual_projected_scaled:.0f}** Covid deaths total.\n')
+    f.write('\n')
+```
+
+```python
+with open('covid_summary.md', 'a') as f:
+    f.write('\n')
+    f.write('## Deaths doubling times\n')
+    f.write(f'Based on a 7-day moving average\n')
+    f.write('\n')
+    f.write('![Deaths doubling times](covid_doubling_times_7.png)\n')
+    f.write('\n')
+```
+
+```python
+with open('covid_summary.md', 'a') as f:
+    f.write('\n')
+    f.write('## Cases per day and lockdown dates\n')
+    f.write(f'Based on a 7-day moving average\n')
+    f.write('\n')
+    f.write('![Cases per day](cases_per_day_with_lockdown.png)\n')
+    f.write('\n')
+```
+
+```python
+with open('covid_summary.md', 'a') as f:
+    f.write('\n')
+    f.write('| Country ID | Country name | Most recent daily cases | Most recent daily deaths |\n')
+    f.write('|:-----------|:-------------|------------------------:|-------------------------:|\n')
+    for c in sorted(COUNTRIES_CORE):
+        lvic = cases_m7[c].last_valid_index()
+        lvid = deaths_m7[c].last_valid_index()
+        f.write(f'| {c} | {countries.loc[c].countriesAndTerritories} | {cases_m7[c][lvic]:.0f} | {deaths_m7[c][lvid]:.0f} | \n')
+    f.write('\n')
+    f.write('(Figures are 7-day averages)\n')
+    f.write('\n')
+```
+
+```python
+with open('covid_summary.md', 'a') as f:
+    f.write('# Data sources\n')
+    f.write('\n')
+    f.write('> Covid data from [European Centre for Disease Prevention and Control](https://www.ecdc.europa.eu/en/publications-data/download-todays-data-geographic-distribution-covid-19-cases-worldwide)\n')
+    f.write('\n')    
+    f.write("""> Population data from:
+
+* [Office of National Statistics](https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/deaths/datasets/weeklyprovisionalfiguresondeathsregisteredinenglandandwales) (Endland and Wales) Weeks start on a Saturday.
+* [Northern Ireland Statistics and Research Agency](https://www.nisra.gov.uk/publications/weekly-deaths) (Northern Ireland). Weeks start on a Saturday. Note that the week numbers don't match the England and Wales data.
+* [National Records of Scotland](https://www.nrscotland.gov.uk/statistics-and-data/statistics/statistics-by-theme/vital-events/general-publications/weekly-and-monthly-data-on-births-and-deaths/weekly-data-on-births-and-deaths) (Scotland). Note that Scotland uses ISO8601 week numbers, which start on a Monday.""")
+    
+    f.write('\n\n')
+    f.write('> [Source code available](https://git.njae.me.uk/?p=covid19.git;a=tree)\n')
+    f.write('\n') 
+
+```
+
+```python
+!pandoc -s covid_summary.md > covid_summary.html
+```
+
+```python
+!scp covid_summary.html neil@ogedei:/var/www/scripts.njae.me.uk/covid/index.html
+!scp covid_deaths_total_linear.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp deaths-radar.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp covid_deaths_per_day_7.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp covid_doubling_times_7.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp cases_per_day_with_lockdown.png neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+```
+
+```python
+with open('uk_covid_deaths.js', 'w') as f:
+    f.write(f"document.write('{uk_covid_deaths}');")
+    
+with open('estimated_total_deaths.js', 'w') as f:
+    f.write(f"document.write('{uk_deaths_to_date:.0f}');")
+
+with open('projection_date.js', 'w') as f:
+    f.write(f"document.write(\'{projection_date.strftime('%d %B %Y')}\');")
+
+with open('projected_deaths.js', 'w') as f:
+    f.write(f"document.write('{uk_projection.deaths.sum():.0f}');")
+
+with open('projected_excess_deaths.js', 'w') as f:
+    f.write(f"document.write('{deaths_actual_projected_scaled:.0f}');")
+
+with open('excess_deaths_upto.js', 'w') as f:
+    f.write(f"document.write('{pd.to_datetime(excess_deaths_upto).strftime('%d %B %Y')}');")
+
+with open('excess_deaths.js', 'w') as f:
+    f.write(f"document.write('{excess_deaths:.0f}');")
+    
+with open('reported_deaths.js', 'w') as f:
+    f.write(f"document.write('{ons_reported_deaths:.0f}');")
+    
+with open('scaling_factor.js', 'w') as f:
+    f.write(f"document.write('{excess_death_accuracy:.2f}');")  
+
+with open('projection_length.js', 'w') as f:
+    f.write(f"document.write('{s_end - s_start - 1}');")
+    
+with open('s_end.js', 'w') as f:
+    f.write(f"document.write('{s_end}');")
+    
+s_start_date_str = s_start_date.strftime("%d %B %Y")
+with open('s_start_date.js', 'w') as f:
+    f.write(f"document.write('{s_start_date_str}');")
+    
+with open('uk_end.js', 'w') as f:
+    f.write(f"document.write('{uk_end}');")
+    
+with open('last_uk_date.js', 'w') as f:
+    f.write(f"document.write('{pd.to_datetime(last_uk_date).strftime('%d %B %Y')}');")
+```
+
+```python
+pd.to_datetime(excess_deaths_upto).strftime('%d %B %Y')
+```
+
+```python
+!scp uk_covid_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp estimated_total_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp projection_date.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp projected_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp projected_excess_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp excess_deaths_upto.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp excess_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp reported_deaths.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp scaling_factor.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp projection_length.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp s_end.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp s_start_date.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp uk_end.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/
+!scp last_uk_date.js neil@ogedei:/var/www/scripts.njae.me.uk/covid/    
+```
+
+```python
+data_by_date.loc['UK'].to_csv('data_by_day_uk.csv', header=True, index=True)
+```
+
+```python
+ukd = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), ['deaths', 'deaths_m7']].droplevel(1)
+ax = ukd.deaths.plot.bar(figsize=(12, 8))
+ukd.deaths_m7.plot.line(ax=ax, color='red')
+# ax = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), 'deaths_m7'].plot.line(figsize=(12, 8), color='red')
+# ax = data_since_threshold.replace([np.inf, -np.inf], np.nan).loc[(slice(None), ['UK']), 'deaths'].plot.bar(ax=ax)
+ax.set_xticks(range(0, 120, 20))
+```
+
+```python
+np.arange(0, 130, 20)
+```
+
+```python
+data_by_date.loc['UK']
+```
+
+```python
+data_by_date.loc['UK'].plot(x='deaths_culm', y='deaths', logx=True, logy=True)
+```
+
+```python
+data_by_date.loc['UK'].plot(x='cases_culm', y='cases')
+```
+
+```python
+ukdbd = data_by_date.loc['UK'].copy()
+ukdbd['deaths_m7'] = ukdbd.deaths.transform(lambda x: x.rolling(7, 1).mean())
+ukdbd['cases_m7'] = ukdbd.cases.transform(lambda x: x.rolling(7, 1).mean())
+ukdbd
+```
+
+```python
+ukdbd.plot(x='deaths_culm', y='deaths_m7', logx=True, logy=True)
+```
+
+```python
+fig, ax = plt.subplots(figsize=(12, 8))
+xmax = 10
+for c in COUNTRIES_CORE:
+    if data_since_threshold.loc[(slice(None), c), 'deaths_culm'].max() > xmax:
+        xmax = data_since_threshold.loc[(slice(None), c), 'deaths_culm'].max()
+    data_since_threshold.loc[(slice(None), c), :].plot(x='deaths_culm', y='deaths_m7', logx=True, logy=True, xlim=(10, xmax * 1.1), label=c, ax=ax)
+```
+
+```python
+data_since_threshold.loc[(slice(None), 'UK'), 'deaths_culm'].max()
+```
+
+```python
+countries.continentExp.unique()
+```
+
+```python
+countries.loc['KW']
+```
+
+```python
+data_by_date.groupby(level=0)['deaths'].shift(-25)
+```
+
+```python
+offset_data = data_by_date.loc[:, ['cases']]
+offset_data['deaths'] = data_by_date.groupby(level=0)['deaths'].shift(-25)
+offset_data['cases_m7'] = offset_data.groupby(level=0)['cases'].transform(lambda x: x.rolling(7, 1).mean())
+offset_data['deaths_m7'] = offset_data['deaths'].dropna().groupby(level=0).transform(lambda x: x.rolling(7, 1).mean())
+offset_data['deaths_per_case'] = offset_data.deaths_m7 / offset_data.cases_m7
+offset_data
+```
+
+```python
+deaths_m7
+```
+
+```python
+offset_deaths_m7 = (offset_data.loc[COUNTRIES_ALL, ['deaths_m7']]
+             .unstack().sort_index().xs('deaths_m7', axis=1, drop_level=True)).T.sort_index()
+offset_deaths_m7
+```
+
+```python
+offset_deaths_m7['UK']
+```
+
+```python
+data_since_threshold.loc[(slice(None), 'UK'), :].tail()
+```
+
+```python
+countries.loc['PT']
+```
+
+```python
+ax = cases_m7.iloc[-50:][COUNTRIES_FRIENDS].plot(figsize=(15, 9), title="Cases per day, 7 day moving average", ylim=(-10, 1500))
+ax.set_xlabel(f"Days since {DEATH_COUNT_THRESHOLD} deaths")
+# uk_projection.deaths_m7.plot(ax=ax)
+for c in COUNTRIES_FRIENDS:
+    lvi = cases_m7[c].last_valid_index()
+    if c != 'ES':
+        ax.text(x = lvi + 1, y = cases_m7[c][lvi], s = f"{c}: {cases_m7[c][lvi]:.0f}")
+
+```
+
+```python
+ax = deaths_m7.iloc[-50:][COUNTRIES_FRIENDS].plot(figsize=(15, 9), title="Cases per day, 7 day moving average", ylim=(-10, 100))
+ax.set_xlabel(f"Days since {DEATH_COUNT_THRESHOLD} deaths")
+# uk_projection.deaths_m7.plot(ax=ax)
+for c in COUNTRIES_FRIENDS:
+    lvi = deaths_m7[c].last_valid_index()
+#     if c != 'ES':
+    ax.text(x = lvi + 1, y = deaths_m7[c][lvi], s = f"{c}: {deaths_m7[c][lvi]:.0f}")
+
 ```
 
 ```python