test_and_case_data.py

   1 # ---
   2 # jupyter:
   3 #   jupytext:
   4 #     formats: ipynb,py:percent
   5 #     text_representation:
   6 #       extension: .py
   7 #       format_name: percent
   8 #       format_version: '1.3'
   9 #       jupytext_version: 1.10.2
  10 #   kernelspec:
  11 #     display_name: Python 3
  12 #     language: python
  13 #     name: python3
  14 # ---
  15
  16 # %% Collapsed="false"
  17 import itertools
  18 import collections
  19 import json
  20 import pandas as pd
  21 import numpy as np
  22 from scipy.stats import gmean
  23 import datetime
  24
  25 import sqlalchemy
  26
  27 import matplotlib as mpl
  28 import matplotlib.pyplot as plt
  29 import matplotlib.animation as ani
  30 # # %matplotlib inline
  31 # # %load_ext sql
  32 plt.ioff()
  33
  34 # %% Collapsed="false"
  35 chart_start_date = '2020-09-15'
  36
  37 # %% Collapsed="false"
  38 connection_string = 'postgresql://covid:3NbjJTkT63@localhost/covid'
  39
  40 # %% Collapsed="false"
  41 # %sql $connection_string
  42
  43 # %%
  44 engine = sqlalchemy.create_engine(connection_string)
  45
  46 # %%
  47 qstr = '''select uk_data.date,
  48     uk_data.new_cases, uk_data_7.new_cases as new_cases_7,
  49     uk_data.new_tests, uk_data_7.new_tests as new_tests_7,
  50     uk_data.new_pcr_tests, uk_data_7.new_pcr_tests as new_pcr_7,
  51     uk_data.new_pillar_1_2_tests as new_pillar_12, uk_data_7.new_pillar_1_2_tests as new_pillar_12_7,
  52      uk_data.new_cases::float / uk_data.new_tests as fraction_positive,
  53         uk_data_7.new_cases / uk_data_7.new_tests as fraction_positive_7
  54     from uk_data left outer join uk_data_7 using (date)
  55     order by uk_data.date'''
  56 tests_data = pd.read_sql_query(qstr, engine,
  57     index_col='date',
  58     parse_dates = ['date'])
  59
  60 # %%
  61 tests_data.tail(10)
  62
  63 # %% Collapsed="false"
  64 # tests_data[['new_tests', 'new_cases']].plot()
  65
  66 # %% Collapsed="false"
  67 # tests_data[['fraction_positive', 'fraction_positive_7']].dropna().plot()
  68
  69 # %% Collapsed="false"
  70 # ax = data_by_day.dropna().loc['2020-06-15': , ['fraction_positive', 'fraction_positive_m7']].plot(figsize=(10, 8), title='Fraction of tests with positive results')
  71 # ax.legend(['Fraction positive per day', 'Fraction positive, 7 day moving average'])
  72 # ax.set_ylabel('Fraction positive')
  73 # plt.savefig('fraction_positive_tests.png')
  74
  75 # %% Collapsed="false"
  76 # pri_y_max = int((tests_data.dropna().loc['2020-06-15': , 'new_tests_7'].max() * 1.1) / 100 ) * 100
  77 # ax = tests_data.dropna().loc['2020-06-15': , 'new_tests_7'].plot(figsize=(10, 8),
  78 #                                                                style=['k-'],
  79 #                                                                legend=False,
  80 #                                                                ylim=(0, pri_y_max))
  81 # ax.set_title('Tests done and new cases (7 day moving average)')
  82 # ax.legend(['Tests, 7 day moving average'], loc='lower left')
  83 # ax.set_ylabel('Tests')
  84 # sec_y_max = int((tests_data.dropna().loc['2020-06-15':, 'new_cases_7'].max() * 1.1) / 100) * 100
  85 # ax = tests_data.dropna().loc['2020-06-15':, 'new_cases_7'].plot(ax=ax, secondary_y=True, style='r--')
  86 # ax.set_ylim((0, sec_y_max))
  87 # ax.legend(['Cases (7 day moving average)'], loc='lower right')
  88 # ax.set_ylabel('New cases')
  89 # # plt.savefig('tests_and_cases.png')
  90
  91 # %% Collapsed="false"
  92 pri_y_max = int((tests_data.dropna().loc[chart_start_date: , 'new_tests_7'].max() * 1.1) / 100 ) * 100
  93 ax = tests_data.dropna().loc[chart_start_date: , 'new_tests_7'].plot(figsize=(10, 8),
  94                                                                style=['k-'],
  95                                                                legend=False,
  96                                                                ylim=(0, pri_y_max))
  97 ax.set_title('Tests done and new cases (7 day moving average)')
  98 ax.legend(['Tests, 7 day moving average'], loc='lower left')
  99 ax.set_ylabel('Tests')
 100 sec_y_max = int((tests_data.dropna().loc[chart_start_date:, 'new_cases_7'].max() * 1.1) / 100) * 100
 101 ax = tests_data.dropna().loc[chart_start_date:, 'new_cases_7'].plot(ax=ax, secondary_y=True, style='r--')
 102 ax.set_ylim((0, sec_y_max))
 103 ax.legend(['Cases (7 day moving average)'], loc='lower right')
 104 ax.set_ylabel('New cases')
 105 plt.savefig('tests_and_cases.png')
 106
 107 # %% Collapsed="false"
 108 pri_y_max = int((tests_data.dropna().loc[chart_start_date: , 'fraction_positive_7'].max() * 1.1) * 100)
 109
 110 ax = (tests_data.loc[chart_start_date: ,
 111                      ['fraction_positive', 'fraction_positive_7']] * 100).plot(
 112   figsize=(10, 8), style=['b:', 'k-'], legend=False, ylim=(0, pri_y_max))
 113 ax.set_title('Fraction of tests with positive results')
 114 ax.legend(['Fraction positive (%)', 'Fraction positive (%), 7 day moving average'], loc='upper left')
 115 ax.set_ylabel('Fraction positive')
 116 cases_axis_max = (1.1 * tests_data.loc[chart_start_date:].new_cases_7.max()
 117 #                   * tests_data.loc[chart_start_date:].fraction_positive.max()
 118 #                   / tests_data.loc[chart_start_date:].fraction_positive_7.max()
 119                   )
 120
 121 ax2 = ax.twinx()
 122 ax2 = tests_data.loc[chart_start_date:, 'new_cases_7'].plot(ax=ax2, secondary_y=True, style='r--')
 123 ax2.set_ylim(-1000, cases_axis_max)
 124 ax2.legend(['Cases (7 day moving average)'], loc='center left')
 125 ax2.set_ylabel('New cases')
 126 plt.savefig('fraction_positive_tests.png')
 127
 128 # %% Collapsed="false"
 129 ax = tests_data.dropna().loc[chart_start_date:].plot(x='fraction_positive_7', y='new_tests_7',
 130                                                   figsize=(8, 8),
 131                                                   legend=None)
 132 ax.set_xlabel("Fraction of tests that are positive")
 133 ax.set_ylabel("Number of tests")
 134 for d in tests_data.dropna().loc[chart_start_date::15].index:
 135     ax.plot(tests_data.loc[d, 'fraction_positive_7'], tests_data.loc[d, 'new_tests_7'], 'o',
 136                         markersize=8)#, markerfacecolor=marker_col, markeredgecolor=marker_col)
 137     ax.text(tests_data.loc[d, 'fraction_positive_7'] + 0.0002, tests_data.loc[d, 'new_tests_7'],
 138             s = d.strftime("%d %B %Y"))
 139 plt.savefig('fraction_positive_tests_vs_tests.png')
 140
 141 # %% Collapsed="false"
 142 fig = plt.figure(figsize=(8, 8))
 143 plt.ylabel('Number of tests')
 144 plt.xlabel('Fraction of tests that are positive')
 145 all_data = tests_data.dropna().loc[chart_start_date:]
 146
 147
 148 minx = all_data.fraction_positive_7.min() * 0.9
 149 maxx = all_data.fraction_positive_7.max() * 1.1
 150 miny = all_data.new_tests_7.min() * 0.9
 151 maxy = all_data.new_tests_7.max() * 1.1
 152
 153 plt.xlim(minx, maxx)
 154 plt.ylim(miny, maxy)
 155 # plt.legend(None)
 156
 157 def build_state_frame(i):
 158     this_data = all_data[:i]
 159     plt.clf()
 160     plt.ylabel('Number of tests')
 161     plt.xlabel('Fraction of tests that are positive')
 162     plt.xlim(minx, maxx)
 163     plt.ylim(miny, maxy)
 164     p = plt.plot(this_data.fraction_positive_7, this_data.new_tests_7)
 165     p[0].set_color('r')
 166     for d in this_data[::15].index:
 167         plt.plot(this_data.loc[d, 'fraction_positive_7'],
 168                 this_data.loc[d, 'new_tests_7'], 'o',
 169                 markersize=8, markerfacecolor='r', markeredgecolor='r')
 170         plt.text(this_data.loc[d, 'fraction_positive_7'] + 0.0002,
 171                 this_data.loc[d, 'new_tests_7'],
 172             s = d.strftime("%d %B %Y"))
 173
 174 animator = ani.FuncAnimation(fig, build_state_frame,
 175                              frames=all_data.shape[0]+1,
 176                              interval=100,
 177                              repeat_delay=200,
 178                              repeat=True)
 179 animator.save('tests_vs_fraction_positive.mp4')
 180 # plt.show()
 181
 182
 183 # %% Collapsed="false"
 184 # !rm tests_vs_fraction_positive_animation.png
 185 # !ffmpeg -i tests_vs_fraction_positive.mp4 -plays 0 -final_delay 1 -f apng tests_vs_fraction_positive_animation.png
 186
 187 # %% Collapsed="false"
 188 # fig = plt.figure(figsize=(8, 8))
 189 # plt.ylabel('Number of tests')
 190 # plt.xlabel('Fraction of tests that are positive')
 191
 192 # all_data = data_by_day.dropna().loc[chart_start_date:]
 193
 194 # minx = all_data.fraction_positive_m7.min() * 0.9
 195 # maxx = all_data.fraction_positive_m7.max() * 1.1
 196 # miny = all_data.tests_m7.min() * 0.9
 197 # maxy = all_data.tests_m7.max() * 1.1
 198
 199 # plt.xlim(minx, maxx)
 200 # plt.ylim(miny, maxy)
 201 # # plt.legend(None)
 202
 203 # def build_state_frame(i):
 204 #     this_data = all_data[:i]
 205 #     p = plt.plot(this_data.fraction_positive_m7, this_data.tests_m7)
 206 #     p[0].set_color('r')
 207 #     for d in this_data[::15].index:
 208 #         plt.plot(this_data.loc[d, 'fraction_positive_m7'],
 209 #                 this_data.loc[d, 'tests_m7'], 'o',
 210 #                 markersize=8, markerfacecolor='r', markeredgecolor='r')
 211 #         plt.text(this_data.loc[d, 'fraction_positive_m7'] + 0.0002,
 212 #                 this_data.loc[d, 'tests_m7'],
 213 #             s = d.strftime("%d %B %Y"))
 214
 215 # # animator = ani.FuncAnimation(fig, build_state_frame, interval=200)
 216 # # animator.save('myfirstAnimation.mp4')
 217 # build_state_frame(103)
 218 # plt.show()
 219
 220 # %% Collapsed="false"