From: Neil Smith Date: Tue, 29 Mar 2016 15:11:22 +0000 (+0100) Subject: Got the initial set into the database X-Git-Url: https://git.njae.me.uk/?a=commitdiff_plain;h=4ab690d3677c721d3d7770cd583dedc90f0a061c;p=dmarc.git Got the initial set into the database --- diff --git a/complete.ipynb b/complete.ipynb new file mode 100644 index 0000000..25659d3 --- /dev/null +++ b/complete.ipynb @@ -0,0 +1,939 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import configparser\n", + "import imaplib\n", + "import email\n", + "import io\n", + "import zipfile\n", + "import xml.etree.ElementTree\n", + "import psycopg2\n", + "import re\n", + "import datetime" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def fetch_msg(num):\n", + " return mailbox.fetch(num, '(RFC822)')[1][0][1]" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def xml_of_part(part):\n", + " with zipfile.ZipFile(io.BytesIO(part.get_payload(decode=True))) as zf:\n", + " fn = zf.infolist()[0].filename\n", + " contents = zf.read(fn).decode('utf-8')\n", + " return xml.etree.ElementTree.fromstring(contents)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def xml_of(message):\n", + " reports = []\n", + " if message.is_multipart():\n", + " for p in message.get_payload():\n", + " if 'zip' in p.get_content_type():\n", + " reports += [xml_of_part(p)]\n", + " else:\n", + " reports = [xml_of_part(message)]\n", + " return reports" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def extract_report(msg):\n", + " pmsg = email.message_from_bytes(msg)\n", + " return xml_of(pmsg)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['dmarc.ini']" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config = configparser.ConfigParser()\n", + "config.read('dmarc.ini')" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('OK', [b'174'])" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mailbox = imaplib.IMAP4(host=config['imap']['server'], \n", + " port=config['imap']['port'])\n", + "mailbox.starttls()\n", + "mailbox.login(config['imap']['username'], config['imap']['password'])\n", + "mailbox.select('INBOX', readonly=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('OK',\n", + " [b'1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174'])" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "resp, nums = mailbox.search(None, \"ALL\")\n", + "resp, nums" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'1'\n", + "b'1' []\n", + "b'2'\n", + "b'2' []\n", + "b'3'\n", + "b'3' []\n", + "b'4'\n", + "b'4' []\n", + "b'5'\n", + "b'5' []\n", + "b'6'\n", + "b'6' []\n", + "b'7'\n", + "b'7' []\n", + "b'8'\n", + "b'8' []\n", + "b'9'\n", + "b'9' []\n", + "b'10'\n", + "b'10' []\n", + "b'11'\n", + "b'11' []\n", + "b'12'\n", + "b'12' []\n", + "b'13'\n", + "b'13' []\n", + "b'14'\n", + "b'14' []\n", + "b'15'\n", + "b'15' []\n", + "b'16'\n", + "b'16' []\n", + "b'17'\n", + "b'17' []\n", + "b'18'\n", + "b'18' []\n", + "b'19'\n", + "b'19' []\n", + "b'20'\n", + "b'20' []\n", + "b'21'\n", + "b'21' []\n", + "b'22'\n", + "b'22' []\n", + "b'23'\n", + "b'23' []\n", + "b'24'\n", + "b'24' []\n", + "b'25'\n", + "b'25' []\n", + "b'26'\n", + "b'26' []\n", + "b'27'\n", + "b'27' []\n", + "b'28'\n", + "b'28' []\n", + "b'29'\n", + "b'29' []\n", + "b'30'\n", + "b'30' []\n", + "b'31'\n", + "b'31' []\n", + "b'32'\n", + "b'32' []\n", + "b'33'\n", + "b'33' []\n", + "b'34'\n", + "b'34' []\n", + "b'35'\n", + "b'35' []\n", + "b'36'\n", + "b'36' []\n", + "b'37'\n", + "b'37' []\n", + "b'38'\n", + "b'38' []\n", + "b'39'\n", + "b'39' []\n", + "b'40'\n", + "b'40' []\n", + "b'41'\n", + "b'41' []\n", + "b'42'\n", + "b'42' []\n", + "b'43'\n", + "b'43' []\n", + "b'44'\n", + "b'44' []\n", + "b'45'\n", + "b'45' []\n", + "b'46'\n", + "b'46' []\n", + "b'47'\n", + "b'47' []\n", + "b'48'\n", + "b'48' []\n", + "b'49'\n", + "b'49' []\n", + "b'50'\n", + "b'50' []\n", + "b'51'\n", + "b'51' []\n", + "b'52'\n", + "b'52' []\n", + "b'53'\n", + "b'53' []\n", + "b'54'\n", + "b'54' []\n", + "b'55'\n", + "b'55' []\n", + "b'56'\n", + "b'56' []\n", + "b'57'\n", + "b'57' []\n", + "b'58'\n", + "b'58' []\n", + "b'59'\n", + "b'59' []\n", + "b'60'\n", + "b'60' []\n", + "b'61'\n", + "b'61' []\n", + "b'62'\n", + "b'62' []\n", + "b'63'\n", + "b'63' []\n", + "b'64'\n", + "b'64' []\n", + "b'65'\n", + "b'65' []\n", + "b'66'\n", + "b'66' []\n", + "b'67'\n", + "b'67' []\n", + "b'68'\n", + "b'68' []\n", + "b'69'\n", + "b'69' []\n", + "b'70'\n", + "b'70' []\n", + "b'71'\n", + "b'71' []\n", + "b'72'\n", + "b'72' []\n", + "b'73'\n", + "b'73' []\n", + "b'74'\n", + "b'74' []\n", + "b'75'\n", + "b'75' []\n", + "b'76'\n", + "b'76' []\n", + "b'77'\n", + "b'77' []\n", + "b'78'\n", + "b'78' []\n", + "b'79'\n", + "b'79' []\n", + "b'80'\n", + "b'80' []\n", + "b'81'\n", + "b'81' []\n", + "b'82'\n", + "b'82' []\n", + "b'83'\n", + "b'83' []\n", + "b'84'\n", + "b'84' []\n", + "b'85'\n", + "b'85' []\n", + "b'86'\n", + "b'86' []\n", + "b'87'\n", + "b'87' []\n", + "b'88'\n", + "b'88' []\n", + "b'89'\n", + "b'89' []\n", + "b'90'\n", + "b'90' []\n", + "b'91'\n", + "b'91' []\n", + "b'92'\n", + "b'92' []\n", + "b'93'\n", + "b'93' []\n", + "b'94'\n", + "b'94' []\n", + "b'95'\n", + "b'95' []\n", + "b'96'\n", + "b'96' []\n", + "b'97'\n", + "b'97' []\n", + "b'98'\n", + "b'98' []\n", + "b'99'\n", + "b'99' []\n", + "b'100'\n", + "b'100' []\n", + "b'101'\n", + "b'101' []\n", + "b'102'\n", + "b'102' []\n", + "b'103'\n", + "b'103' []\n", + "b'104'\n", + "b'104' []\n", + "b'105'\n", + "b'105' []\n", + "b'106'\n", + "b'106' []\n", + "b'107'\n", + "b'107' []\n", + "b'108'\n", + "b'108' []\n", + "b'109'\n", + "b'109' []\n", + "b'110'\n", + "b'110' []\n", + "b'111'\n", + "b'111' []\n", + "b'112'\n", + "b'112' []\n", + "b'113'\n", + "b'113' []\n", + "b'114'\n", + "b'114' []\n", + "b'115'\n", + "b'115' []\n", + "b'116'\n", + "b'116' []\n", + "b'117'\n", + "b'117' []\n", + "b'118'\n", + "b'118' []\n", + "b'119'\n", + "b'119' []\n", + "b'120'\n", + "b'120' []\n", + "b'121'\n", + "b'121' []\n", + "b'122'\n", + "b'122' []\n", + "b'123'\n", + "b'123' []\n", + "b'124'\n", + "b'124' []\n", + "b'125'\n", + "b'125' []\n", + "b'126'\n", + "b'126' []\n", + "b'127'\n", + "b'127' []\n", + "b'128'\n", + "b'128' []\n", + "b'129'\n", + "b'129' []\n", + "b'130'\n", + "b'130' []\n", + "b'131'\n", + "b'131' []\n", + "b'132'\n", + "b'132' []\n", + "b'133'\n", + "b'133' []\n", + "b'134'\n", + "b'134' []\n", + "b'135'\n", + "b'135' []\n", + "b'136'\n", + "b'136' []\n", + "b'137'\n", + "b'137' []\n", + "b'138'\n", + "b'138' []\n", + "b'139'\n", + "b'139' []\n", + "b'140'\n", + "b'140' []\n", + "b'141'\n", + "b'141' []\n", + "b'142'\n", + "b'142' []\n", + "b'143'\n", + "b'143' []\n", + "b'144'\n", + "b'144' []\n", + "b'145'\n", + "b'145' []\n", + "b'146'\n", + "b'146' []\n", + "b'147'\n", + "b'147' []\n", + "b'148'\n", + "b'148' []\n", + "b'149'\n", + "b'149' []\n", + "b'150'\n", + "b'150' []\n", + "b'151'\n", + "b'151' []\n", + "b'152'\n", + "b'152' []\n", + "b'153'\n", + "b'153' []\n", + "b'154'\n", + "b'154' []\n", + "b'155'\n", + "b'155' []\n", + "b'156'\n", + "b'156' []\n", + "b'157'\n", + "b'157' []\n", + "b'158'\n", + "b'158' []\n", + "b'159'\n", + "b'159' []\n", + "b'160'\n", + "b'160' []\n", + "b'161'\n", + "b'161' []\n", + "b'162'\n", + "b'162' []\n", + "b'163'\n", + "b'163' []\n", + "b'164'\n", + "b'164' []\n", + "b'165'\n", + "b'165' []\n", + "b'166'\n", + "b'166' []\n", + "b'167'\n", + "b'167' []\n", + "b'168'\n", + "b'168' []\n", + "b'169'\n", + "b'169' []\n", + "b'170'\n", + "b'170' []\n", + "b'171'\n", + "b'171' []\n", + "b'172'\n", + "b'172' []\n", + "b'173'\n", + "b'173' []\n", + "b'174'\n", + "b'174' []\n" + ] + } + ], + "source": [ + "for n in nums[0].split():\n", + " print(n)\n", + " reports = extract_report(fetch_msg(n))\n", + " print(n, reports)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ,\n", + " ]" + ] + }, + "execution_count": 45, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dmarc_reports = [report for report_set in [extract_report(fetch_msg(n)) for n in nums[0].split()]\n", + " for report in report_set]\n", + "dmarc_reports" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "('BYE', [b'Logging out'])" + ] + }, + "execution_count": 46, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mailbox.close()\n", + "mailbox.logout()" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "field_maps = {'./policy_published/adkim': {'pg_field_name': 'policy_published_adkim',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './policy_published/aspf': {'pg_field_name': 'policy_published_aspf',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './policy_published/domain': {'pg_field_name': 'policy_published_domain',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './policy_published/p': {'pg_field_name': 'policy_published_p',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './policy_published/pct': {'pg_field_name': 'policy_published_pct',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'int'},\n", + " './record[{}]/auth_results/dkim/domain': {'pg_field_name': 'auth_results_dkim_domain',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/auth_results/dkim/result': {'pg_field_name': 'auth_results_dkim_result',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/auth_results/spf/domain': {'pg_field_name': 'auth_results_spf_domain',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/auth_results/spf/result': {'pg_field_name': 'auth_results_spf_result',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/identifiers/header_from': {'pg_field_name': 'identifiers_header_from',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/row/count': {'pg_field_name': 'count',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'int'},\n", + " './record[{}]/row/policy_evaluated/disposition': {'pg_field_name': 'policy_evaluated_disposition',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/row/policy_evaluated/dkim': {'pg_field_name': 'policy_evaluated_dkim',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/row/policy_evaluated/spf': {'pg_field_name': 'policy_evaluated_spf',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/row/source_ip': {'pg_field_name': 'source_ip',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'inet'},\n", + " './report_metadata/date_range/begin': {'pg_field_name': 'report_metadata_date_range_begin',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'timestamp'},\n", + " './report_metadata/date_range/end': {'pg_field_name': 'report_metadata_date_range_end',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'timestamp'},\n", + " './report_metadata/email': {'pg_field_name': 'report_metadata_email',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './report_metadata/org_name': {'pg_field_name': 'report_metadata_org_name',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './report_metadata/report_id': {'pg_field_name': 'report_metadata_report_id',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'}}" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def maybe_strip(text):\n", + " if text:\n", + " return text.strip()\n", + " else:\n", + " return ''" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def write_report(connection, cursor, report):\n", + " \n", + " field_names = []\n", + " values = {}\n", + " for f in [f for f in field_maps if field_maps[f]['pg_table'] == 'reports']:\n", + " field_names += [field_maps[f]['pg_field_name']]\n", + " if field_maps[f]['pg_type'] == 'int':\n", + " values[field_maps[f]['pg_field_name']] = int(report.find(f).text)\n", + " elif field_maps[f]['pg_type'] == 'timestamp':\n", + " values[field_maps[f]['pg_field_name']] = datetime.datetime.utcfromtimestamp(int(report.find(f).text))\n", + " elif field_maps[f]['pg_type'] == 'inet':\n", + " values[field_maps[f]['pg_field_name']] = maybe_strip(report.find(f).text)\n", + " else:\n", + " values[field_maps[f]['pg_field_name']] = maybe_strip(report.find(f).text)\n", + " insert_string = 'insert into reports (' + ', '.join(field_names) + ') '\n", + " insert_string += 'values (' + ', '.join('%({})s'.format(f) for f in field_names) + ');'\n", + " cursor.execute(insert_string, values)\n", + " \n", + " for i in range(1, len(report.findall('./record'))+1):\n", + " field_names = []\n", + " cursor.execute('select id, report_metadata_report_id from reports where report_metadata_report_id = %s;', \n", + " [report.find('./report_metadata/report_id').text])\n", + " results = cursor.fetchall()\n", + " if len(results) != 1:\n", + " raise\n", + " else:\n", + " report_id = results[0][0]\n", + " values = {'report_id': report_id}\n", + " for f in [f for f in field_maps if field_maps[f]['pg_table'] == 'report_items']:\n", + " field_names += [field_maps[f]['pg_field_name']]\n", + " if field_maps[f]['pg_type'] == 'int':\n", + " values[field_maps[f]['pg_field_name']] = int(report.find(f.format(i)).text)\n", + " elif field_maps[f]['pg_type'] == 'timestamp':\n", + " values[field_maps[f]['pg_field_name']] = datetime.datetime.utcfromtimestamp(int(report.find(f.format(i)).text))\n", + " elif field_maps[f]['pg_type'] == 'inet':\n", + " values[field_maps[f]['pg_field_name']] = maybe_strip(report.find(f.format(i)).text)\n", + " else:\n", + " values[field_maps[f]['pg_field_name']] = maybe_strip(report.find(f.format(i)).text)\n", + " insert_string = 'insert into report_items (report_id, ' + ', '.join(field_names) + ') '\n", + " insert_string += 'values (%(report_id)s, ' + ', '.join('%({})s'.format(f) for f in field_names) + ');'\n", + " cursor.execute(insert_string, values)\n", + " connection.commit()" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "conn = psycopg2.connect(host=config['database']['server'],\n", + " database=config['database']['database'], \n", + " user=config['database']['username'], \n", + " password=config['database']['password'])\n", + "with conn.cursor() as cur:\n", + " for report in dmarc_reports:\n", + " write_report(conn, cur, report)" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "conn.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.4.3+" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/make_database.ipynb b/make_database.ipynb new file mode 100644 index 0000000..895bed8 --- /dev/null +++ b/make_database.ipynb @@ -0,0 +1,238 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import configparser\n", + "import psycopg2" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "field_maps = {'./policy_published/adkim': {'pg_field_name': 'policy_published_adkim',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './policy_published/aspf': {'pg_field_name': 'policy_published_aspf',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './policy_published/domain': {'pg_field_name': 'policy_published_domain',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './policy_published/p': {'pg_field_name': 'policy_published_p',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './policy_published/pct': {'pg_field_name': 'policy_published_pct',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'int'},\n", + " './record[{}]/auth_results/dkim/domain': {'pg_field_name': 'auth_results_dkim_domain',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/auth_results/dkim/result': {'pg_field_name': 'auth_results_dkim_result',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/auth_results/spf/domain': {'pg_field_name': 'auth_results_spf_domain',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/auth_results/spf/result': {'pg_field_name': 'auth_results_spf_result',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/identifiers/header_from': {'pg_field_name': 'identifiers_header_from',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/row/count': {'pg_field_name': 'count',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'int'},\n", + " './record[{}]/row/policy_evaluated/disposition': {'pg_field_name': 'policy_evaluated_disposition',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/row/policy_evaluated/dkim': {'pg_field_name': 'policy_evaluated_dkim',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/row/policy_evaluated/spf': {'pg_field_name': 'policy_evaluated_spf',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'varchar'},\n", + " './record[{}]/row/source_ip': {'pg_field_name': 'source_ip',\n", + " 'pg_table': 'report_items',\n", + " 'pg_type': 'inet'},\n", + " './report_metadata/date_range/begin': {'pg_field_name': 'report_metadata_date_range_begin',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'timestamp'},\n", + " './report_metadata/date_range/end': {'pg_field_name': 'report_metadata_date_range_end',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'timestamp'},\n", + " './report_metadata/email': {'pg_field_name': 'report_metadata_email',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './report_metadata/org_name': {'pg_field_name': 'report_metadata_org_name',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'},\n", + " './report_metadata/report_id': {'pg_field_name': 'report_metadata_report_id',\n", + " 'pg_table': 'reports',\n", + " 'pg_type': 'varchar'}}" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['dmarc.ini']" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config = configparser.ConfigParser()\n", + "config.read('dmarc.ini')" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "conn = psycopg2.connect(host=config['database']['server'],\n", + " database=config['database']['database'], \n", + " user=config['database']['username'], \n", + " password=config['database']['password'])" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "with conn.cursor() as cur:\n", + " conn.autocommit = True\n", + " cur.execute(\"select exists(select * from information_schema.tables where table_name=%s)\", ('report_items',))\n", + " if cur.fetchone()[0]:\n", + " cur.execute(\"drop table report_items;\")\n", + " cur.execute(\"select exists(select * from information_schema.tables where table_name=%s)\", ('reports',))\n", + " if cur.fetchone()[0]:\n", + " cur.execute(\"drop table reports;\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'create table reports (id serial primary key, policy_published_pct int, policy_published_adkim varchar, report_metadata_org_name varchar, policy_published_aspf varchar, policy_published_domain varchar, report_metadata_date_range_end timestamp, policy_published_p varchar, report_metadata_report_id varchar, report_metadata_email varchar, report_metadata_date_range_begin timestamp);'" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "create_report_table_string = 'create table reports (id serial primary key, ' + \\\n", + "', '.join(field_maps[p]['pg_field_name'] + ' ' + field_maps[p]['pg_type'] \n", + " for p in field_maps if field_maps[p]['pg_table'] == 'reports') + \\\n", + "');'\n", + "create_report_table_string" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'create table report_items (id serial primary key, report_id integer references reports, auth_results_spf_domain varchar, source_ip inet, policy_evaluated_spf varchar, count int, auth_results_dkim_result varchar, auth_results_dkim_domain varchar, policy_evaluated_disposition varchar, identifiers_header_from varchar, policy_evaluated_dkim varchar, auth_results_spf_result varchar);'" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "create_report_item_table_string = 'create table report_items (id serial primary key, ' + \\\n", + " 'report_id integer references reports, ' + \\\n", + "', '.join(field_maps[p]['pg_field_name'] + ' ' + field_maps[p]['pg_type'] \n", + " for p in field_maps if field_maps[p]['pg_table'] == 'report_items') + \\\n", + "');'\n", + "create_report_item_table_string" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "with conn.cursor() as cur:\n", + " cur.execute(create_report_table_string)\n", + " cur.execute(create_report_item_table_string)\n", + "conn.commit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.4.3+" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/read_report.ipynb b/read_report.ipynb index b5bf2df..0108e30 100644 --- a/read_report.ipynb +++ b/read_report.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 40, "metadata": { "collapsed": false }, @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 41, "metadata": { "collapsed": false }, @@ -32,7 +32,7 @@ "['dmarc.ini']" ] }, - "execution_count": 2, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -44,7 +44,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 42, "metadata": { "collapsed": false }, @@ -55,7 +55,7 @@ "[]" ] }, - "execution_count": 3, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 43, "metadata": { "collapsed": false }, @@ -77,7 +77,7 @@ "['imap', 'database']" ] }, - "execution_count": 31, + "execution_count": 43, "metadata": {}, "output_type": "execute_result" } @@ -88,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 44, "metadata": { "collapsed": false }, @@ -99,7 +99,7 @@ "'imap.njae.me.uk'" ] }, - "execution_count": 4, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -110,7 +110,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 45, "metadata": { "collapsed": false }, @@ -122,7 +122,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 46, "metadata": { "collapsed": false }, @@ -133,7 +133,7 @@ "('OK', [None])" ] }, - "execution_count": 6, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -144,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 47, "metadata": { "collapsed": false }, @@ -155,7 +155,7 @@ "('OK', [b'Logged in'])" ] }, - "execution_count": 7, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -166,7 +166,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 48, "metadata": { "collapsed": false }, @@ -174,10 +174,10 @@ { "data": { "text/plain": [ - "('OK', [b'172'])" + "('OK', [b'174'])" ] }, - "execution_count": 8, + "execution_count": 48, "metadata": {}, "output_type": "execute_result" } @@ -188,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 49, "metadata": { "collapsed": false }, @@ -197,10 +197,10 @@ "data": { "text/plain": [ "('OK',\n", - " [b'1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172'])" + " [b'1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174'])" ] }, - "execution_count": 9, + "execution_count": 49, "metadata": {}, "output_type": "execute_result" } @@ -211,7 +211,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 50, "metadata": { "collapsed": false }, @@ -219,10 +219,10 @@ { "data": { "text/plain": [ - "('OK', [b'164 165 166 167 168 169 170 171 172'])" + "('OK', [b'164 165 166 167 168 169 170 171 172 173 174'])" ] }, - "execution_count": 10, + "execution_count": 50, "metadata": {}, "output_type": "execute_result" } @@ -234,7 +234,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 51, "metadata": { "collapsed": false }, @@ -242,10 +242,20 @@ { "data": { "text/plain": [ - "[b'164', b'165', b'166', b'167', b'168', b'169', b'170', b'171', b'172']" + "[b'164',\n", + " b'165',\n", + " b'166',\n", + " b'167',\n", + " b'168',\n", + " b'169',\n", + " b'170',\n", + " b'171',\n", + " b'172',\n", + " b'173',\n", + " b'174']" ] }, - "execution_count": 11, + "execution_count": 51, "metadata": {}, "output_type": "execute_result" } @@ -256,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 52, "metadata": { "collapsed": false }, @@ -265,12 +275,12 @@ "data": { "text/plain": [ "('OK',\n", - " [(b'172 (RFC822 {4406}',\n", - " b'Return-Path: \\r\\nDelivered-To: dmarc@njae.me.uk\\r\\nReceived: from localhost (localhost [127.0.0.1])\\r\\n\\tby mail.njae.me.uk (Postfix) with ESMTP id B966C8F\\r\\n\\tfor ; Mon, 28 Mar 2016 11:11:48 +0100 (BST)\\r\\nX-Virus-Scanned: Debian amavisd-new at njae.me.uk\\r\\nAuthentication-Results: ogedei.njae.me.uk (amavisd-new);\\r\\n\\tdkim=pass (2048-bit key) header.d=google.com\\r\\nReceived: from mail.njae.me.uk ([127.0.0.1])\\r\\n\\tby localhost (ogedei.njae.me.uk [127.0.0.1]) (amavisd-new, port 10024)\\r\\n\\twith ESMTP id GOe6I1LLVNl9 for ;\\r\\n\\tMon, 28 Mar 2016 11:11:47 +0100 (BST)\\r\\nReceived-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=209.85.223.202; helo=mail-io0-f202.google.com; envelope-from=noreply-dmarc-support@google.com; receiver=dmarc@njae.me.uk \\r\\nAuthentication-Results: mail.njae.me.uk; dmarc=pass header.from=google.com\\r\\nAuthentication-Results: mail.njae.me.uk; dkim=pass\\r\\n\\treason=\"2048-bit key; unprotected key\"\\r\\n\\theader.d=google.com header.i=@google.com header.b=FskZAi7J;\\r\\n\\tdkim-adsp=pass; dkim-atps=neutral\\r\\nReceived: from mail-io0-f202.google.com (mail-io0-f202.google.com [209.85.223.202])\\r\\n\\tby mail.njae.me.uk (Postfix) with ESMTPS id CDAA850\\r\\n\\tfor ; Mon, 28 Mar 2016 11:11:46 +0100 (BST)\\r\\nReceived: by mail-io0-f202.google.com with SMTP id e3so1655636ioa.1\\r\\n for ; Mon, 28 Mar 2016 03:11:46 -0700 (PDT)\\r\\nDKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;\\r\\n d=google.com; s=20120113;\\r\\n h=mime-version:message-id:date:subject:from:to:content-disposition\\r\\n :content-transfer-encoding;\\r\\n bh=ZMmMsYb7psOB4wlBTQUv+llYFZLud7uqkJWOqTZoywc=;\\r\\n b=FskZAi7J5MJ/Py4sgZaL4dHbNcmccPHDWlFnblx0RT0Dlfb35hDkjtYg5ageqyFowe\\r\\n o4hDspB8+L7cBsY+ftdbMhneveFpKvDgwZqq/88iMOWLuUN39ZWMC7m/XIz2CZhUlLMz\\r\\n xE25EhSOW9xiWWBOESZzIfCvSOoTuyfxMTUFq1uvARg2NUOAdrFTLTivAsMyK2RNkisM\\r\\n SrQRcwDaMLIlQw6FzG+gsO3u8wrT6ckd0fHJtkGptab6CqhEwdkY3kJMCmiasoP/io2v\\r\\n KEsQCkByjHJgnL4tFu5yznLggBrbbvdK4HonSGKDi7cHyuoImxaVZNVMQhaYRnyPqj7j\\r\\n o08g==\\r\\nX-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;\\r\\n d=1e100.net; s=20130820;\\r\\n h=x-gm-message-state:mime-version:message-id:date:subject:from:to\\r\\n :content-disposition:content-transfer-encoding;\\r\\n bh=ZMmMsYb7psOB4wlBTQUv+llYFZLud7uqkJWOqTZoywc=;\\r\\n b=OJjK6bq/e//bsrZJ503gIDT0b582UN5hgfMRT+iJVwtH7ombD4ld+S/+NOfWVTwq8J\\r\\n FoHVodNZjznUtkoOVOm3BFbliMPyxHoYKdbWyYgYH5KzlD9Iv81N5djl50nxdWNkbhK1\\r\\n CZHike9tfLHF82v6i8BIRMubK8UHyCCOpVgFMM+GK3tTdyA7BY/0jXNqzfTDbS/FyeoW\\r\\n sH/sckfOf4HN9DRgNbZ78aq5c1bAcx/N7Fn7+53uLBxw82XUd7JFi10kMd3jvPbHMORa\\r\\n YQgyPoQxuQPvjfU0KTJiXQt+iP+ksHl0DO9umo+c5WzLdHk8QV1ryS5OEhUrfaORqK8a\\r\\n 0d9g==\\r\\nX-Gm-Message-State: AD7BkJLC5599FbOXiJoIKq3UlGXdkIr5BkghJuDHCJeDAQVhAr64/XYgB+fejr5uDj8FvDHPPSPVXA==\\r\\nMIME-Version: 1.0\\r\\nX-Received: by 10.182.243.135 with SMTP id wy7mr16780454obc.8.1459159905131;\\r\\n Mon, 28 Mar 2016 03:11:45 -0700 (PDT)\\r\\nMessage-ID: <16143280651570354241@google.com>\\r\\nDate: Mon, 28 Mar 2016 10:11:45 +0000\\r\\nSubject: Report domain: njae.me.uk Submitter: google.com Report-ID: 16143280651570354241\\r\\nFrom: noreply-dmarc-support@google.com\\r\\nTo: dmarc@njae.me.uk\\r\\nContent-Type: application/zip; \\r\\n\\tname=\"google.com!njae.me.uk!1459036800!1459123199.zip\"\\r\\nContent-Disposition: attachment; \\r\\n\\tfilename=\"google.com!njae.me.uk!1459036800!1459123199.zip\"\\r\\nContent-Transfer-Encoding: base64\\r\\n\\r\\nUEsDBAoAAAAIAD1PfEjhndhx2QEAAHoEAAAvAAAAZ29vZ2xlLmNvbSFuamFlLm1lLnVrITE0NTkw\\r\\nMzY4MDAhMTQ1OTEyMzE5OS54bWytVMtyozAQvOcrKN+NEAbW3lKUPeULds+ULAasNUgqSeTx9yss\\r\\n8YiTqlz2hOiZ6ZnuEZCnt6FPXsBYoeTjDqfZLgHJVSNk97j78/t5f9wlT/SBtADNmfErfUgSYkAr\\r\\n4+oBHGuYYxPmUWW6WrIBaKdU10PK1UDQAoYcGJjoqVSeoX/fNwMzfG9HPdH92paFvFjz5gyruZKO\\r\\ncVcL2Sp6cU7bnwjF0nQtRQwxaV/BoLyoqvKYea7P9YE4yhANxRUuDvkxq0pc/sgOZZEXmKA1HvK9\\r\\nVqgNk11U46EzdEJSXJSn7FAdM98tIHMcZHOL4vyATyc/i5zJ0Ee2pdvWVKJVL/h7rcdzL+wFlkGU\\r\\nt0dS+ZdBOkA6Xj1bgEKcNVcxUENQOETQ6vaGTc8Aab8JCQTp+G5nwM6I5o7iSdZ0uI351UjeSa7M\\r\\nPJ1Rr4t+q0bDoRaa5jhPq1NalmmV+wYLPmdyNUpHfSgcZji2gxfWj96wZg5MLgirlRXO39w49hbZ\\r\\n5E0eaGatT1jsiHrbGFg82Ui86+k3NAsjogHpRCv8d7OUXYA1YOrWqOHDZrZ4JPpUTtjoLrUBO/Zu\\r\\nZbyb9putxys9UURR8WXRteUjG8X/g3u9VOhey5QcrgdB62/kH1BLAQIKAAoAAAAIAD1PfEjhndhx\\r\\n2QEAAHoEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIW5qYWUubWUudWshMTQ1OTAzNjgw\\r\\nMCExNDU5MTIzMTk5LnhtbFBLBQYAAAAAAQABAF0AAAAmAgAAAAA=\\r\\n'),\n", + " [(b'174 (RFC822 {4399}',\n", + " b'Return-Path: \\r\\nDelivered-To: dmarc@njae.me.uk\\r\\nReceived: from localhost (localhost [127.0.0.1])\\r\\n\\tby mail.njae.me.uk (Postfix) with ESMTP id 38B4132B\\r\\n\\tfor ; Tue, 29 Mar 2016 10:19:13 +0100 (BST)\\r\\nX-Virus-Scanned: Debian amavisd-new at njae.me.uk\\r\\nAuthentication-Results: ogedei.njae.me.uk (amavisd-new);\\r\\n\\tdkim=pass (2048-bit key) header.d=google.com\\r\\nReceived: from mail.njae.me.uk ([127.0.0.1])\\r\\n\\tby localhost (ogedei.njae.me.uk [127.0.0.1]) (amavisd-new, port 10024)\\r\\n\\twith ESMTP id SnLdON04KYCo for ;\\r\\n\\tTue, 29 Mar 2016 10:19:10 +0100 (BST)\\r\\nReceived-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=209.85.213.73; helo=mail-vk0-f73.google.com; envelope-from=noreply-dmarc-support@google.com; receiver=dmarc@njae.me.uk \\r\\nAuthentication-Results: mail.njae.me.uk; dmarc=pass header.from=google.com\\r\\nAuthentication-Results: mail.njae.me.uk; dkim=pass\\r\\n\\treason=\"2048-bit key; unprotected key\"\\r\\n\\theader.d=google.com header.i=@google.com header.b=iAXipzAt;\\r\\n\\tdkim-adsp=pass; dkim-atps=neutral\\r\\nReceived: from mail-vk0-f73.google.com (mail-vk0-f73.google.com [209.85.213.73])\\r\\n\\tby mail.njae.me.uk (Postfix) with ESMTPS id 179E41E5\\r\\n\\tfor ; Tue, 29 Mar 2016 10:19:09 +0100 (BST)\\r\\nReceived: by mail-vk0-f73.google.com with SMTP id z68so702703vkg.0\\r\\n for ; Tue, 29 Mar 2016 02:19:09 -0700 (PDT)\\r\\nDKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;\\r\\n d=google.com; s=20120113;\\r\\n h=mime-version:message-id:date:subject:from:to:content-disposition\\r\\n :content-transfer-encoding;\\r\\n bh=LH2uamJGGOiwcPQ0Y/w6LUkyxCGuLyHT3APu5+Zs63I=;\\r\\n b=iAXipzAtsCytlhTKIQ8CQy7SgB9q5HZ71k1QE93vubKwtldTgCdIZc3KiZLhJA0TLw\\r\\n 5YEVf+rfppwog3YJhffpeZgb+O8oaCxAga47sMGxSq3He0bP8TpHtVo+EaJyRaFVbOei\\r\\n X2f4/WuC1gF5eDg/JYhJgIkq0owFlhl10CgLnFRZCEWIFNuuPKLmhWrIceqZY5Cay90X\\r\\n CCmEBsFYY+cvfnbtzhJeSLG8w9vHbyCZVs8cY5XFOfaRmcWmKH/Yq/fpcAZ9BaMYme2U\\r\\n YW1lSKKFlFf5aJtaO2YGQMPyEZqLRsDjFo+wldyTMwaptG5hyYMH1MRTrRuCbUpsIV8t\\r\\n mHag==\\r\\nX-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;\\r\\n d=1e100.net; s=20130820;\\r\\n h=x-gm-message-state:mime-version:message-id:date:subject:from:to\\r\\n :content-disposition:content-transfer-encoding;\\r\\n bh=LH2uamJGGOiwcPQ0Y/w6LUkyxCGuLyHT3APu5+Zs63I=;\\r\\n b=V5D9qH2AnRN1Yl9Q/ZaOCk7+NDT8Uk2kJRGxFu8goO+F1oJfRVe5zyp9Ig2DTAwUaj\\r\\n 8odegu+0RiGm3hrRYlbcTw3Mdsa3D2Gf2GWxAfEWTsheH2D/8cv6W8XqsFUhNl4w7j/f\\r\\n 9m4dQ6bBCqX2sBoD/vu+3dvtiL1u7qe4QiBFYlZy/75hbPJzaPkaGbXJyFl9jJLBmmR8\\r\\n DTMIwevCzUgHpoh0ITYL7lAW640jKV9ouNcXSo+pfsarztbhC8bBG+5Lh9SigI1EqgFB\\r\\n cUgZ+Oxuzkq93CcYHWTYYr1tsO7Y55w/+5DsAhiqjgPCkFIGrbLLguUM2+9V6yYex1v2\\r\\n JgXw==\\r\\nX-Gm-Message-State: AD7BkJLvHE6gfQN3L4TDuBK40mhasmyPR5+mR27t8PtP3Ym5H6IsuU0eB5uXi4w8dar7jVodeLR/5A==\\r\\nMIME-Version: 1.0\\r\\nX-Received: by 10.129.87.74 with SMTP id l71mr605278ywb.38.1459243148486; Tue,\\r\\n 29 Mar 2016 02:19:08 -0700 (PDT)\\r\\nMessage-ID: <15410706527896810898@google.com>\\r\\nDate: Tue, 29 Mar 2016 09:19:08 +0000\\r\\nSubject: Report domain: njae.me.uk Submitter: google.com Report-ID: 15410706527896810898\\r\\nFrom: noreply-dmarc-support@google.com\\r\\nTo: dmarc@njae.me.uk\\r\\nContent-Type: application/zip; \\r\\n\\tname=\"google.com!njae.me.uk!1459123200!1459209599.zip\"\\r\\nContent-Disposition: attachment; \\r\\n\\tfilename=\"google.com!njae.me.uk!1459123200!1459209599.zip\"\\r\\nContent-Transfer-Encoding: base64\\r\\n\\r\\nUEsDBAoAAAAIAAlJfUgBY8K+2QEAAHoEAAAvAAAAZ29vZ2xlLmNvbSFuamFlLm1lLnVrITE0NTkx\\r\\nMjMyMDAhMTQ1OTIwOTU5OS54bWytVE2TnCAQve+vsOY+ou7oaoplc8ovSM4WA61DRoEC3I9/Hxzw\\r\\nY2dTlUtO4uvu1/1eo/jlfRySVzBWKPl8yNPskIBkigvZPx9+/fxxrA/JC3nAHQA/U3YlD0mCDWhl\\r\\nXDuCo5w6OmMeVaZvJR2B9Er1A6RMjRitYMiBkYqBSOUZho8jH6lhRzvpme77vizkxZp3Z2jLlHSU\\r\\nuVbITpGLc9p+QyiWplspoohK+wYGFaeqKuvMc32tD8RRhuAkL0959pRVZfFUN1WdZ3VTY7TFQ77X\\r\\nCq2hso9qPHSGXkiSn8omLx6LzHcLyBIHyW/RImvKpvGzyIUMfWZbu+1NxVoNgn20ejoPwl5gHUR5\\r\\neySRvymkI6TT1bMFKMQpv4qRGIzCIYJWdzdsfgZI+01IwEjHd7sAdkE0cySfZc2H25h/G8k7yZRZ\\r\\npjPqbdVv1WQYtEKTIi/SqknLMq0K32DFl0ymJunII0bhsMCxHbzSYfKG8SUwuyCsVlY4f3Pj2Htk\\r\\nlzd7oKm1PmG1I+rtYmD1ZCfxrqff0CIMCw7SiU7472YtuwDlYNrOqPHTZvZ4JPpSjunkLq0BOw1u\\r\\nY7yb9h9bj1d6poii4suqa8+Hd4r/B/d2qdC9ljk5XA+Mtt/IH1BLAQIKAAoAAAAIAAlJfUgBY8K+\\r\\n2QEAAHoEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIW5qYWUubWUudWshMTQ1OTEyMzIw\\r\\nMCExNDU5MjA5NTk5LnhtbFBLBQYAAAAAAQABAF0AAAAmAgAAAAA=\\r\\n'),\n", " b')'])" ] }, - "execution_count": 12, + "execution_count": 52, "metadata": {}, "output_type": "execute_result" } @@ -282,7 +292,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 53, "metadata": { "collapsed": false }, @@ -294,75 +304,75 @@ "Return-Path: \r\n", "Delivered-To: dmarc@njae.me.uk\r\n", "Received: from localhost (localhost [127.0.0.1])\r\n", - "\tby mail.njae.me.uk (Postfix) with ESMTP id B966C8F\r\n", - "\tfor ; Mon, 28 Mar 2016 11:11:48 +0100 (BST)\r\n", + "\tby mail.njae.me.uk (Postfix) with ESMTP id 38B4132B\r\n", + "\tfor ; Tue, 29 Mar 2016 10:19:13 +0100 (BST)\r\n", "X-Virus-Scanned: Debian amavisd-new at njae.me.uk\r\n", "Authentication-Results: ogedei.njae.me.uk (amavisd-new);\r\n", "\tdkim=pass (2048-bit key) header.d=google.com\r\n", "Received: from mail.njae.me.uk ([127.0.0.1])\r\n", "\tby localhost (ogedei.njae.me.uk [127.0.0.1]) (amavisd-new, port 10024)\r\n", - "\twith ESMTP id GOe6I1LLVNl9 for ;\r\n", - "\tMon, 28 Mar 2016 11:11:47 +0100 (BST)\r\n", - "Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=209.85.223.202; helo=mail-io0-f202.google.com; envelope-from=noreply-dmarc-support@google.com; receiver=dmarc@njae.me.uk \r\n", + "\twith ESMTP id SnLdON04KYCo for ;\r\n", + "\tTue, 29 Mar 2016 10:19:10 +0100 (BST)\r\n", + "Received-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=209.85.213.73; helo=mail-vk0-f73.google.com; envelope-from=noreply-dmarc-support@google.com; receiver=dmarc@njae.me.uk \r\n", "Authentication-Results: mail.njae.me.uk; dmarc=pass header.from=google.com\r\n", "Authentication-Results: mail.njae.me.uk; dkim=pass\r\n", "\treason=\"2048-bit key; unprotected key\"\r\n", - "\theader.d=google.com header.i=@google.com header.b=FskZAi7J;\r\n", + "\theader.d=google.com header.i=@google.com header.b=iAXipzAt;\r\n", "\tdkim-adsp=pass; dkim-atps=neutral\r\n", - "Received: from mail-io0-f202.google.com (mail-io0-f202.google.com [209.85.223.202])\r\n", - "\tby mail.njae.me.uk (Postfix) with ESMTPS id CDAA850\r\n", - "\tfor ; Mon, 28 Mar 2016 11:11:46 +0100 (BST)\r\n", - "Received: by mail-io0-f202.google.com with SMTP id e3so1655636ioa.1\r\n", - " for ; Mon, 28 Mar 2016 03:11:46 -0700 (PDT)\r\n", + "Received: from mail-vk0-f73.google.com (mail-vk0-f73.google.com [209.85.213.73])\r\n", + "\tby mail.njae.me.uk (Postfix) with ESMTPS id 179E41E5\r\n", + "\tfor ; Tue, 29 Mar 2016 10:19:09 +0100 (BST)\r\n", + "Received: by mail-vk0-f73.google.com with SMTP id z68so702703vkg.0\r\n", + " for ; Tue, 29 Mar 2016 02:19:09 -0700 (PDT)\r\n", "DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;\r\n", " d=google.com; s=20120113;\r\n", " h=mime-version:message-id:date:subject:from:to:content-disposition\r\n", " :content-transfer-encoding;\r\n", - " bh=ZMmMsYb7psOB4wlBTQUv+llYFZLud7uqkJWOqTZoywc=;\r\n", - " b=FskZAi7J5MJ/Py4sgZaL4dHbNcmccPHDWlFnblx0RT0Dlfb35hDkjtYg5ageqyFowe\r\n", - " o4hDspB8+L7cBsY+ftdbMhneveFpKvDgwZqq/88iMOWLuUN39ZWMC7m/XIz2CZhUlLMz\r\n", - " xE25EhSOW9xiWWBOESZzIfCvSOoTuyfxMTUFq1uvARg2NUOAdrFTLTivAsMyK2RNkisM\r\n", - " SrQRcwDaMLIlQw6FzG+gsO3u8wrT6ckd0fHJtkGptab6CqhEwdkY3kJMCmiasoP/io2v\r\n", - " KEsQCkByjHJgnL4tFu5yznLggBrbbvdK4HonSGKDi7cHyuoImxaVZNVMQhaYRnyPqj7j\r\n", - " o08g==\r\n", + " bh=LH2uamJGGOiwcPQ0Y/w6LUkyxCGuLyHT3APu5+Zs63I=;\r\n", + " b=iAXipzAtsCytlhTKIQ8CQy7SgB9q5HZ71k1QE93vubKwtldTgCdIZc3KiZLhJA0TLw\r\n", + " 5YEVf+rfppwog3YJhffpeZgb+O8oaCxAga47sMGxSq3He0bP8TpHtVo+EaJyRaFVbOei\r\n", + " X2f4/WuC1gF5eDg/JYhJgIkq0owFlhl10CgLnFRZCEWIFNuuPKLmhWrIceqZY5Cay90X\r\n", + " CCmEBsFYY+cvfnbtzhJeSLG8w9vHbyCZVs8cY5XFOfaRmcWmKH/Yq/fpcAZ9BaMYme2U\r\n", + " YW1lSKKFlFf5aJtaO2YGQMPyEZqLRsDjFo+wldyTMwaptG5hyYMH1MRTrRuCbUpsIV8t\r\n", + " mHag==\r\n", "X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;\r\n", " d=1e100.net; s=20130820;\r\n", " h=x-gm-message-state:mime-version:message-id:date:subject:from:to\r\n", " :content-disposition:content-transfer-encoding;\r\n", - " bh=ZMmMsYb7psOB4wlBTQUv+llYFZLud7uqkJWOqTZoywc=;\r\n", - " b=OJjK6bq/e//bsrZJ503gIDT0b582UN5hgfMRT+iJVwtH7ombD4ld+S/+NOfWVTwq8J\r\n", - " FoHVodNZjznUtkoOVOm3BFbliMPyxHoYKdbWyYgYH5KzlD9Iv81N5djl50nxdWNkbhK1\r\n", - " CZHike9tfLHF82v6i8BIRMubK8UHyCCOpVgFMM+GK3tTdyA7BY/0jXNqzfTDbS/FyeoW\r\n", - " sH/sckfOf4HN9DRgNbZ78aq5c1bAcx/N7Fn7+53uLBxw82XUd7JFi10kMd3jvPbHMORa\r\n", - " YQgyPoQxuQPvjfU0KTJiXQt+iP+ksHl0DO9umo+c5WzLdHk8QV1ryS5OEhUrfaORqK8a\r\n", - " 0d9g==\r\n", - "X-Gm-Message-State: AD7BkJLC5599FbOXiJoIKq3UlGXdkIr5BkghJuDHCJeDAQVhAr64/XYgB+fejr5uDj8FvDHPPSPVXA==\r\n", + " bh=LH2uamJGGOiwcPQ0Y/w6LUkyxCGuLyHT3APu5+Zs63I=;\r\n", + " b=V5D9qH2AnRN1Yl9Q/ZaOCk7+NDT8Uk2kJRGxFu8goO+F1oJfRVe5zyp9Ig2DTAwUaj\r\n", + " 8odegu+0RiGm3hrRYlbcTw3Mdsa3D2Gf2GWxAfEWTsheH2D/8cv6W8XqsFUhNl4w7j/f\r\n", + " 9m4dQ6bBCqX2sBoD/vu+3dvtiL1u7qe4QiBFYlZy/75hbPJzaPkaGbXJyFl9jJLBmmR8\r\n", + " DTMIwevCzUgHpoh0ITYL7lAW640jKV9ouNcXSo+pfsarztbhC8bBG+5Lh9SigI1EqgFB\r\n", + " cUgZ+Oxuzkq93CcYHWTYYr1tsO7Y55w/+5DsAhiqjgPCkFIGrbLLguUM2+9V6yYex1v2\r\n", + " JgXw==\r\n", + "X-Gm-Message-State: AD7BkJLvHE6gfQN3L4TDuBK40mhasmyPR5+mR27t8PtP3Ym5H6IsuU0eB5uXi4w8dar7jVodeLR/5A==\r\n", "MIME-Version: 1.0\r\n", - "X-Received: by 10.182.243.135 with SMTP id wy7mr16780454obc.8.1459159905131;\r\n", - " Mon, 28 Mar 2016 03:11:45 -0700 (PDT)\r\n", - "Message-ID: <16143280651570354241@google.com>\r\n", - "Date: Mon, 28 Mar 2016 10:11:45 +0000\r\n", - "Subject: Report domain: njae.me.uk Submitter: google.com Report-ID: 16143280651570354241\r\n", + "X-Received: by 10.129.87.74 with SMTP id l71mr605278ywb.38.1459243148486; Tue,\r\n", + " 29 Mar 2016 02:19:08 -0700 (PDT)\r\n", + "Message-ID: <15410706527896810898@google.com>\r\n", + "Date: Tue, 29 Mar 2016 09:19:08 +0000\r\n", + "Subject: Report domain: njae.me.uk Submitter: google.com Report-ID: 15410706527896810898\r\n", "From: noreply-dmarc-support@google.com\r\n", "To: dmarc@njae.me.uk\r\n", "Content-Type: application/zip; \r\n", - "\tname=\"google.com!njae.me.uk!1459036800!1459123199.zip\"\r\n", + "\tname=\"google.com!njae.me.uk!1459123200!1459209599.zip\"\r\n", "Content-Disposition: attachment; \r\n", - "\tfilename=\"google.com!njae.me.uk!1459036800!1459123199.zip\"\r\n", + "\tfilename=\"google.com!njae.me.uk!1459123200!1459209599.zip\"\r\n", "Content-Transfer-Encoding: base64\r\n", "\r\n", - "UEsDBAoAAAAIAD1PfEjhndhx2QEAAHoEAAAvAAAAZ29vZ2xlLmNvbSFuamFlLm1lLnVrITE0NTkw\r\n", - "MzY4MDAhMTQ1OTEyMzE5OS54bWytVMtyozAQvOcrKN+NEAbW3lKUPeULds+ULAasNUgqSeTx9yss\r\n", - "8YiTqlz2hOiZ6ZnuEZCnt6FPXsBYoeTjDqfZLgHJVSNk97j78/t5f9wlT/SBtADNmfErfUgSYkAr\r\n", - "4+oBHGuYYxPmUWW6WrIBaKdU10PK1UDQAoYcGJjoqVSeoX/fNwMzfG9HPdH92paFvFjz5gyruZKO\r\n", - "cVcL2Sp6cU7bnwjF0nQtRQwxaV/BoLyoqvKYea7P9YE4yhANxRUuDvkxq0pc/sgOZZEXmKA1HvK9\r\n", - "VqgNk11U46EzdEJSXJSn7FAdM98tIHMcZHOL4vyATyc/i5zJ0Ee2pdvWVKJVL/h7rcdzL+wFlkGU\r\n", - "t0dS+ZdBOkA6Xj1bgEKcNVcxUENQOETQ6vaGTc8Aab8JCQTp+G5nwM6I5o7iSdZ0uI351UjeSa7M\r\n", - "PJ1Rr4t+q0bDoRaa5jhPq1NalmmV+wYLPmdyNUpHfSgcZji2gxfWj96wZg5MLgirlRXO39w49hbZ\r\n", - "5E0eaGatT1jsiHrbGFg82Ui86+k3NAsjogHpRCv8d7OUXYA1YOrWqOHDZrZ4JPpUTtjoLrUBO/Zu\r\n", - "Zbyb9putxys9UURR8WXRteUjG8X/g3u9VOhey5QcrgdB62/kH1BLAQIKAAoAAAAIAD1PfEjhndhx\r\n", - "2QEAAHoEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIW5qYWUubWUudWshMTQ1OTAzNjgw\r\n", - "MCExNDU5MTIzMTk5LnhtbFBLBQYAAAAAAQABAF0AAAAmAgAAAAA=\r\n", + "UEsDBAoAAAAIAAlJfUgBY8K+2QEAAHoEAAAvAAAAZ29vZ2xlLmNvbSFuamFlLm1lLnVrITE0NTkx\r\n", + "MjMyMDAhMTQ1OTIwOTU5OS54bWytVE2TnCAQve+vsOY+ou7oaoplc8ovSM4WA61DRoEC3I9/Hxzw\r\n", + "Y2dTlUtO4uvu1/1eo/jlfRySVzBWKPl8yNPskIBkigvZPx9+/fxxrA/JC3nAHQA/U3YlD0mCDWhl\r\n", + "XDuCo5w6OmMeVaZvJR2B9Er1A6RMjRitYMiBkYqBSOUZho8jH6lhRzvpme77vizkxZp3Z2jLlHSU\r\n", + "uVbITpGLc9p+QyiWplspoohK+wYGFaeqKuvMc32tD8RRhuAkL0959pRVZfFUN1WdZ3VTY7TFQ77X\r\n", + "Cq2hso9qPHSGXkiSn8omLx6LzHcLyBIHyW/RImvKpvGzyIUMfWZbu+1NxVoNgn20ejoPwl5gHUR5\r\n", + "eySRvymkI6TT1bMFKMQpv4qRGIzCIYJWdzdsfgZI+01IwEjHd7sAdkE0cySfZc2H25h/G8k7yZRZ\r\n", + "pjPqbdVv1WQYtEKTIi/SqknLMq0K32DFl0ymJunII0bhsMCxHbzSYfKG8SUwuyCsVlY4f3Pj2Htk\r\n", + "lzd7oKm1PmG1I+rtYmD1ZCfxrqff0CIMCw7SiU7472YtuwDlYNrOqPHTZvZ4JPpSjunkLq0BOw1u\r\n", + "Y7yb9h9bj1d6poii4suqa8+Hd4r/B/d2qdC9ljk5XA+Mtt/IH1BLAQIKAAoAAAAIAAlJfUgBY8K+\r\n", + "2QEAAHoEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIW5qYWUubWUudWshMTQ1OTEyMzIw\r\n", + "MCExNDU5MjA5NTk5LnhtbFBLBQYAAAAAAQABAF0AAAAmAgAAAAA=\r\n", "\n" ] } @@ -373,7 +383,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 54, "metadata": { "collapsed": false }, @@ -382,12 +392,12 @@ "data": { "text/plain": [ "('OK',\n", - " [(b'172 (BODY[1] {912}',\n", - " b'UEsDBAoAAAAIAD1PfEjhndhx2QEAAHoEAAAvAAAAZ29vZ2xlLmNvbSFuamFlLm1lLnVrITE0NTkw\\r\\nMzY4MDAhMTQ1OTEyMzE5OS54bWytVMtyozAQvOcrKN+NEAbW3lKUPeULds+ULAasNUgqSeTx9yss\\r\\n8YiTqlz2hOiZ6ZnuEZCnt6FPXsBYoeTjDqfZLgHJVSNk97j78/t5f9wlT/SBtADNmfErfUgSYkAr\\r\\n4+oBHGuYYxPmUWW6WrIBaKdU10PK1UDQAoYcGJjoqVSeoX/fNwMzfG9HPdH92paFvFjz5gyruZKO\\r\\ncVcL2Sp6cU7bnwjF0nQtRQwxaV/BoLyoqvKYea7P9YE4yhANxRUuDvkxq0pc/sgOZZEXmKA1HvK9\\r\\nVqgNk11U46EzdEJSXJSn7FAdM98tIHMcZHOL4vyATyc/i5zJ0Ee2pdvWVKJVL/h7rcdzL+wFlkGU\\r\\nt0dS+ZdBOkA6Xj1bgEKcNVcxUENQOETQ6vaGTc8Aab8JCQTp+G5nwM6I5o7iSdZ0uI351UjeSa7M\\r\\nPJ1Rr4t+q0bDoRaa5jhPq1NalmmV+wYLPmdyNUpHfSgcZji2gxfWj96wZg5MLgirlRXO39w49hbZ\\r\\n5E0eaGatT1jsiHrbGFg82Ui86+k3NAsjogHpRCv8d7OUXYA1YOrWqOHDZrZ4JPpUTtjoLrUBO/Zu\\r\\nZbyb9putxys9UURR8WXRteUjG8X/g3u9VOhey5QcrgdB62/kH1BLAQIKAAoAAAAIAD1PfEjhndhx\\r\\n2QEAAHoEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIW5qYWUubWUudWshMTQ1OTAzNjgw\\r\\nMCExNDU5MTIzMTk5LnhtbFBLBQYAAAAAAQABAF0AAAAmAgAAAAA=\\r\\n'),\n", + " [(b'174 (BODY[1] {912}',\n", + " b'UEsDBAoAAAAIAAlJfUgBY8K+2QEAAHoEAAAvAAAAZ29vZ2xlLmNvbSFuamFlLm1lLnVrITE0NTkx\\r\\nMjMyMDAhMTQ1OTIwOTU5OS54bWytVE2TnCAQve+vsOY+ou7oaoplc8ovSM4WA61DRoEC3I9/Hxzw\\r\\nY2dTlUtO4uvu1/1eo/jlfRySVzBWKPl8yNPskIBkigvZPx9+/fxxrA/JC3nAHQA/U3YlD0mCDWhl\\r\\nXDuCo5w6OmMeVaZvJR2B9Er1A6RMjRitYMiBkYqBSOUZho8jH6lhRzvpme77vizkxZp3Z2jLlHSU\\r\\nuVbITpGLc9p+QyiWplspoohK+wYGFaeqKuvMc32tD8RRhuAkL0959pRVZfFUN1WdZ3VTY7TFQ77X\\r\\nCq2hso9qPHSGXkiSn8omLx6LzHcLyBIHyW/RImvKpvGzyIUMfWZbu+1NxVoNgn20ejoPwl5gHUR5\\r\\neySRvymkI6TT1bMFKMQpv4qRGIzCIYJWdzdsfgZI+01IwEjHd7sAdkE0cySfZc2H25h/G8k7yZRZ\\r\\npjPqbdVv1WQYtEKTIi/SqknLMq0K32DFl0ymJunII0bhsMCxHbzSYfKG8SUwuyCsVlY4f3Pj2Htk\\r\\nlzd7oKm1PmG1I+rtYmD1ZCfxrqff0CIMCw7SiU7472YtuwDlYNrOqPHTZvZ4JPpSjunkLq0BOw1u\\r\\nY7yb9h9bj1d6poii4suqa8+Hd4r/B/d2qdC9ljk5XA+Mtt/IH1BLAQIKAAoAAAAIAAlJfUgBY8K+\\r\\n2QEAAHoEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIW5qYWUubWUudWshMTQ1OTEyMzIw\\r\\nMCExNDU5MjA5NTk5LnhtbFBLBQYAAAAAAQABAF0AAAAmAgAAAAA=\\r\\n'),\n", " b')'])" ] }, - "execution_count": 14, + "execution_count": 54, "metadata": {}, "output_type": "execute_result" } @@ -399,7 +409,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 55, "metadata": { "collapsed": false }, @@ -408,10 +418,10 @@ "data": { "text/plain": [ "('OK',\n", - " [b'172 (BODY (\"application\" \"zip\" (\"name\" \"google.com!njae.me.uk!1459036800!1459123199.zip\") NIL NIL \"base64\" 912))'])" + " [b'174 (BODY (\"application\" \"zip\" (\"name\" \"google.com!njae.me.uk!1459123200!1459209599.zip\") NIL NIL \"base64\" 912))'])" ] }, - "execution_count": 15, + "execution_count": 55, "metadata": {}, "output_type": "execute_result" } @@ -423,7 +433,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 56, "metadata": { "collapsed": false }, @@ -434,7 +444,7 @@ "55" ] }, - "execution_count": 16, + "execution_count": 56, "metadata": {}, "output_type": "execute_result" } @@ -445,7 +455,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 57, "metadata": { "collapsed": false }, @@ -456,7 +466,7 @@ "int" ] }, - "execution_count": 17, + "execution_count": 57, "metadata": {}, "output_type": "execute_result" } @@ -467,7 +477,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 80, "metadata": { "collapsed": false }, @@ -475,10 +485,10 @@ { "data": { "text/plain": [ - "b'Return-Path: \\r\\nDelivered-To: dmarc@njae.me.uk\\r\\nReceived: from localhost (localhost [127.0.0.1])\\r\\n\\tby mail.njae.me.uk (Postfix) with ESMTP id B966C8F\\r\\n\\tfor ; Mon, 28 Mar 2016 11:11:48 +0100 (BST)\\r\\nX-Virus-Scanned: Debian amavisd-new at njae.me.uk\\r\\nAuthentication-Results: ogedei.njae.me.uk (amavisd-new);\\r\\n\\tdkim=pass (2048-bit key) header.d=google.com\\r\\nReceived: from mail.njae.me.uk ([127.0.0.1])\\r\\n\\tby localhost (ogedei.njae.me.uk [127.0.0.1]) (amavisd-new, port 10024)\\r\\n\\twith ESMTP id GOe6I1LLVNl9 for ;\\r\\n\\tMon, 28 Mar 2016 11:11:47 +0100 (BST)\\r\\nReceived-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=209.85.223.202; helo=mail-io0-f202.google.com; envelope-from=noreply-dmarc-support@google.com; receiver=dmarc@njae.me.uk \\r\\nAuthentication-Results: mail.njae.me.uk; dmarc=pass header.from=google.com\\r\\nAuthentication-Results: mail.njae.me.uk; dkim=pass\\r\\n\\treason=\"2048-bit key; unprotected key\"\\r\\n\\theader.d=google.com header.i=@google.com header.b=FskZAi7J;\\r\\n\\tdkim-adsp=pass; dkim-atps=neutral\\r\\nReceived: from mail-io0-f202.google.com (mail-io0-f202.google.com [209.85.223.202])\\r\\n\\tby mail.njae.me.uk (Postfix) with ESMTPS id CDAA850\\r\\n\\tfor ; Mon, 28 Mar 2016 11:11:46 +0100 (BST)\\r\\nReceived: by mail-io0-f202.google.com with SMTP id e3so1655636ioa.1\\r\\n for ; Mon, 28 Mar 2016 03:11:46 -0700 (PDT)\\r\\nDKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;\\r\\n d=google.com; s=20120113;\\r\\n h=mime-version:message-id:date:subject:from:to:content-disposition\\r\\n :content-transfer-encoding;\\r\\n bh=ZMmMsYb7psOB4wlBTQUv+llYFZLud7uqkJWOqTZoywc=;\\r\\n b=FskZAi7J5MJ/Py4sgZaL4dHbNcmccPHDWlFnblx0RT0Dlfb35hDkjtYg5ageqyFowe\\r\\n o4hDspB8+L7cBsY+ftdbMhneveFpKvDgwZqq/88iMOWLuUN39ZWMC7m/XIz2CZhUlLMz\\r\\n xE25EhSOW9xiWWBOESZzIfCvSOoTuyfxMTUFq1uvARg2NUOAdrFTLTivAsMyK2RNkisM\\r\\n SrQRcwDaMLIlQw6FzG+gsO3u8wrT6ckd0fHJtkGptab6CqhEwdkY3kJMCmiasoP/io2v\\r\\n KEsQCkByjHJgnL4tFu5yznLggBrbbvdK4HonSGKDi7cHyuoImxaVZNVMQhaYRnyPqj7j\\r\\n o08g==\\r\\nX-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;\\r\\n d=1e100.net; s=20130820;\\r\\n h=x-gm-message-state:mime-version:message-id:date:subject:from:to\\r\\n :content-disposition:content-transfer-encoding;\\r\\n bh=ZMmMsYb7psOB4wlBTQUv+llYFZLud7uqkJWOqTZoywc=;\\r\\n b=OJjK6bq/e//bsrZJ503gIDT0b582UN5hgfMRT+iJVwtH7ombD4ld+S/+NOfWVTwq8J\\r\\n FoHVodNZjznUtkoOVOm3BFbliMPyxHoYKdbWyYgYH5KzlD9Iv81N5djl50nxdWNkbhK1\\r\\n CZHike9tfLHF82v6i8BIRMubK8UHyCCOpVgFMM+GK3tTdyA7BY/0jXNqzfTDbS/FyeoW\\r\\n sH/sckfOf4HN9DRgNbZ78aq5c1bAcx/N7Fn7+53uLBxw82XUd7JFi10kMd3jvPbHMORa\\r\\n YQgyPoQxuQPvjfU0KTJiXQt+iP+ksHl0DO9umo+c5WzLdHk8QV1ryS5OEhUrfaORqK8a\\r\\n 0d9g==\\r\\nX-Gm-Message-State: AD7BkJLC5599FbOXiJoIKq3UlGXdkIr5BkghJuDHCJeDAQVhAr64/XYgB+fejr5uDj8FvDHPPSPVXA==\\r\\nMIME-Version: 1.0\\r\\nX-Received: by 10.182.243.135 with SMTP id wy7mr16780454obc.8.1459159905131;\\r\\n Mon, 28 Mar 2016 03:11:45 -0700 (PDT)\\r\\nMessage-ID: <16143280651570354241@google.com>\\r\\nDate: Mon, 28 Mar 2016 10:11:45 +0000\\r\\nSubject: Report domain: njae.me.uk Submitter: google.com Report-ID: 16143280651570354241\\r\\nFrom: noreply-dmarc-support@google.com\\r\\nTo: dmarc@njae.me.uk\\r\\nContent-Type: application/zip; \\r\\n\\tname=\"google.com!njae.me.uk!1459036800!1459123199.zip\"\\r\\nContent-Disposition: attachment; \\r\\n\\tfilename=\"google.com!njae.me.uk!1459036800!1459123199.zip\"\\r\\nContent-Transfer-Encoding: base64\\r\\n\\r\\nUEsDBAoAAAAIAD1PfEjhndhx2QEAAHoEAAAvAAAAZ29vZ2xlLmNvbSFuamFlLm1lLnVrITE0NTkw\\r\\nMzY4MDAhMTQ1OTEyMzE5OS54bWytVMtyozAQvOcrKN+NEAbW3lKUPeULds+ULAasNUgqSeTx9yss\\r\\n8YiTqlz2hOiZ6ZnuEZCnt6FPXsBYoeTjDqfZLgHJVSNk97j78/t5f9wlT/SBtADNmfErfUgSYkAr\\r\\n4+oBHGuYYxPmUWW6WrIBaKdU10PK1UDQAoYcGJjoqVSeoX/fNwMzfG9HPdH92paFvFjz5gyruZKO\\r\\ncVcL2Sp6cU7bnwjF0nQtRQwxaV/BoLyoqvKYea7P9YE4yhANxRUuDvkxq0pc/sgOZZEXmKA1HvK9\\r\\nVqgNk11U46EzdEJSXJSn7FAdM98tIHMcZHOL4vyATyc/i5zJ0Ee2pdvWVKJVL/h7rcdzL+wFlkGU\\r\\nt0dS+ZdBOkA6Xj1bgEKcNVcxUENQOETQ6vaGTc8Aab8JCQTp+G5nwM6I5o7iSdZ0uI351UjeSa7M\\r\\nPJ1Rr4t+q0bDoRaa5jhPq1NalmmV+wYLPmdyNUpHfSgcZji2gxfWj96wZg5MLgirlRXO39w49hbZ\\r\\n5E0eaGatT1jsiHrbGFg82Ui86+k3NAsjogHpRCv8d7OUXYA1YOrWqOHDZrZ4JPpUTtjoLrUBO/Zu\\r\\nZbyb9putxys9UURR8WXRteUjG8X/g3u9VOhey5QcrgdB62/kH1BLAQIKAAoAAAAIAD1PfEjhndhx\\r\\n2QEAAHoEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIW5qYWUubWUudWshMTQ1OTAzNjgw\\r\\nMCExNDU5MTIzMTk5LnhtbFBLBQYAAAAAAQABAF0AAAAmAgAAAAA=\\r\\n'" + "b'Return-Path: \\r\\nDelivered-To: dmarc@njae.me.uk\\r\\nReceived: from localhost (localhost [127.0.0.1])\\r\\n\\tby mail.njae.me.uk (Postfix) with ESMTP id 38B4132B\\r\\n\\tfor ; Tue, 29 Mar 2016 10:19:13 +0100 (BST)\\r\\nX-Virus-Scanned: Debian amavisd-new at njae.me.uk\\r\\nAuthentication-Results: ogedei.njae.me.uk (amavisd-new);\\r\\n\\tdkim=pass (2048-bit key) header.d=google.com\\r\\nReceived: from mail.njae.me.uk ([127.0.0.1])\\r\\n\\tby localhost (ogedei.njae.me.uk [127.0.0.1]) (amavisd-new, port 10024)\\r\\n\\twith ESMTP id SnLdON04KYCo for ;\\r\\n\\tTue, 29 Mar 2016 10:19:10 +0100 (BST)\\r\\nReceived-SPF: Pass (sender SPF authorized) identity=mailfrom; client-ip=209.85.213.73; helo=mail-vk0-f73.google.com; envelope-from=noreply-dmarc-support@google.com; receiver=dmarc@njae.me.uk \\r\\nAuthentication-Results: mail.njae.me.uk; dmarc=pass header.from=google.com\\r\\nAuthentication-Results: mail.njae.me.uk; dkim=pass\\r\\n\\treason=\"2048-bit key; unprotected key\"\\r\\n\\theader.d=google.com header.i=@google.com header.b=iAXipzAt;\\r\\n\\tdkim-adsp=pass; dkim-atps=neutral\\r\\nReceived: from mail-vk0-f73.google.com (mail-vk0-f73.google.com [209.85.213.73])\\r\\n\\tby mail.njae.me.uk (Postfix) with ESMTPS id 179E41E5\\r\\n\\tfor ; Tue, 29 Mar 2016 10:19:09 +0100 (BST)\\r\\nReceived: by mail-vk0-f73.google.com with SMTP id z68so702703vkg.0\\r\\n for ; Tue, 29 Mar 2016 02:19:09 -0700 (PDT)\\r\\nDKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;\\r\\n d=google.com; s=20120113;\\r\\n h=mime-version:message-id:date:subject:from:to:content-disposition\\r\\n :content-transfer-encoding;\\r\\n bh=LH2uamJGGOiwcPQ0Y/w6LUkyxCGuLyHT3APu5+Zs63I=;\\r\\n b=iAXipzAtsCytlhTKIQ8CQy7SgB9q5HZ71k1QE93vubKwtldTgCdIZc3KiZLhJA0TLw\\r\\n 5YEVf+rfppwog3YJhffpeZgb+O8oaCxAga47sMGxSq3He0bP8TpHtVo+EaJyRaFVbOei\\r\\n X2f4/WuC1gF5eDg/JYhJgIkq0owFlhl10CgLnFRZCEWIFNuuPKLmhWrIceqZY5Cay90X\\r\\n CCmEBsFYY+cvfnbtzhJeSLG8w9vHbyCZVs8cY5XFOfaRmcWmKH/Yq/fpcAZ9BaMYme2U\\r\\n YW1lSKKFlFf5aJtaO2YGQMPyEZqLRsDjFo+wldyTMwaptG5hyYMH1MRTrRuCbUpsIV8t\\r\\n mHag==\\r\\nX-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;\\r\\n d=1e100.net; s=20130820;\\r\\n h=x-gm-message-state:mime-version:message-id:date:subject:from:to\\r\\n :content-disposition:content-transfer-encoding;\\r\\n bh=LH2uamJGGOiwcPQ0Y/w6LUkyxCGuLyHT3APu5+Zs63I=;\\r\\n b=V5D9qH2AnRN1Yl9Q/ZaOCk7+NDT8Uk2kJRGxFu8goO+F1oJfRVe5zyp9Ig2DTAwUaj\\r\\n 8odegu+0RiGm3hrRYlbcTw3Mdsa3D2Gf2GWxAfEWTsheH2D/8cv6W8XqsFUhNl4w7j/f\\r\\n 9m4dQ6bBCqX2sBoD/vu+3dvtiL1u7qe4QiBFYlZy/75hbPJzaPkaGbXJyFl9jJLBmmR8\\r\\n DTMIwevCzUgHpoh0ITYL7lAW640jKV9ouNcXSo+pfsarztbhC8bBG+5Lh9SigI1EqgFB\\r\\n cUgZ+Oxuzkq93CcYHWTYYr1tsO7Y55w/+5DsAhiqjgPCkFIGrbLLguUM2+9V6yYex1v2\\r\\n JgXw==\\r\\nX-Gm-Message-State: AD7BkJLvHE6gfQN3L4TDuBK40mhasmyPR5+mR27t8PtP3Ym5H6IsuU0eB5uXi4w8dar7jVodeLR/5A==\\r\\nMIME-Version: 1.0\\r\\nX-Received: by 10.129.87.74 with SMTP id l71mr605278ywb.38.1459243148486; Tue,\\r\\n 29 Mar 2016 02:19:08 -0700 (PDT)\\r\\nMessage-ID: <15410706527896810898@google.com>\\r\\nDate: Tue, 29 Mar 2016 09:19:08 +0000\\r\\nSubject: Report domain: njae.me.uk Submitter: google.com Report-ID: 15410706527896810898\\r\\nFrom: noreply-dmarc-support@google.com\\r\\nTo: dmarc@njae.me.uk\\r\\nContent-Type: application/zip; \\r\\n\\tname=\"google.com!njae.me.uk!1459123200!1459209599.zip\"\\r\\nContent-Disposition: attachment; \\r\\n\\tfilename=\"google.com!njae.me.uk!1459123200!1459209599.zip\"\\r\\nContent-Transfer-Encoding: base64\\r\\n\\r\\nUEsDBAoAAAAIAAlJfUgBY8K+2QEAAHoEAAAvAAAAZ29vZ2xlLmNvbSFuamFlLm1lLnVrITE0NTkx\\r\\nMjMyMDAhMTQ1OTIwOTU5OS54bWytVE2TnCAQve+vsOY+ou7oaoplc8ovSM4WA61DRoEC3I9/Hxzw\\r\\nY2dTlUtO4uvu1/1eo/jlfRySVzBWKPl8yNPskIBkigvZPx9+/fxxrA/JC3nAHQA/U3YlD0mCDWhl\\r\\nXDuCo5w6OmMeVaZvJR2B9Er1A6RMjRitYMiBkYqBSOUZho8jH6lhRzvpme77vizkxZp3Z2jLlHSU\\r\\nuVbITpGLc9p+QyiWplspoohK+wYGFaeqKuvMc32tD8RRhuAkL0959pRVZfFUN1WdZ3VTY7TFQ77X\\r\\nCq2hso9qPHSGXkiSn8omLx6LzHcLyBIHyW/RImvKpvGzyIUMfWZbu+1NxVoNgn20ejoPwl5gHUR5\\r\\neySRvymkI6TT1bMFKMQpv4qRGIzCIYJWdzdsfgZI+01IwEjHd7sAdkE0cySfZc2H25h/G8k7yZRZ\\r\\npjPqbdVv1WQYtEKTIi/SqknLMq0K32DFl0ymJunII0bhsMCxHbzSYfKG8SUwuyCsVlY4f3Pj2Htk\\r\\nlzd7oKm1PmG1I+rtYmD1ZCfxrqff0CIMCw7SiU7472YtuwDlYNrOqPHTZvZ4JPpSjunkLq0BOw1u\\r\\nY7yb9h9bj1d6poii4suqa8+Hd4r/B/d2qdC9ljk5XA+Mtt/IH1BLAQIKAAoAAAAIAAlJfUgBY8K+\\r\\n2QEAAHoEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIW5qYWUubWUudWshMTQ1OTEyMzIw\\r\\nMCExNDU5MjA5NTk5LnhtbFBLBQYAAAAAAQABAF0AAAAmAgAAAAA=\\r\\n'" ] }, - "execution_count": 18, + "execution_count": 80, "metadata": {}, "output_type": "execute_result" } @@ -490,7 +500,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 81, "metadata": { "collapsed": false }, @@ -498,10 +508,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 19, + "execution_count": 81, "metadata": {}, "output_type": "execute_result" } @@ -513,7 +523,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 82, "metadata": { "collapsed": false }, @@ -521,10 +531,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 20, + "execution_count": 82, "metadata": {}, "output_type": "execute_result" } @@ -536,7 +546,29 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 87, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 87, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pmsg.is_multipart()" + ] + }, + { + "cell_type": "code", + "execution_count": 85, "metadata": { "collapsed": false }, @@ -547,7 +579,7 @@ "'application/zip'" ] }, - "execution_count": 21, + "execution_count": 85, "metadata": {}, "output_type": "execute_result" } @@ -558,7 +590,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 86, "metadata": { "collapsed": false }, @@ -566,10 +598,10 @@ { "data": { "text/plain": [ - "'UEsDBAoAAAAIAD1PfEjhndhx2QEAAHoEAAAvAAAAZ29vZ2xlLmNvbSFuamFlLm1lLnVrITE0NTkw\\r\\nMzY4MDAhMTQ1OTEyMzE5OS54bWytVMtyozAQvOcrKN+NEAbW3lKUPeULds+ULAasNUgqSeTx9yss\\r\\n8YiTqlz2hOiZ6ZnuEZCnt6FPXsBYoeTjDqfZLgHJVSNk97j78/t5f9wlT/SBtADNmfErfUgSYkAr\\r\\n4+oBHGuYYxPmUWW6WrIBaKdU10PK1UDQAoYcGJjoqVSeoX/fNwMzfG9HPdH92paFvFjz5gyruZKO\\r\\ncVcL2Sp6cU7bnwjF0nQtRQwxaV/BoLyoqvKYea7P9YE4yhANxRUuDvkxq0pc/sgOZZEXmKA1HvK9\\r\\nVqgNk11U46EzdEJSXJSn7FAdM98tIHMcZHOL4vyATyc/i5zJ0Ee2pdvWVKJVL/h7rcdzL+wFlkGU\\r\\nt0dS+ZdBOkA6Xj1bgEKcNVcxUENQOETQ6vaGTc8Aab8JCQTp+G5nwM6I5o7iSdZ0uI351UjeSa7M\\r\\nPJ1Rr4t+q0bDoRaa5jhPq1NalmmV+wYLPmdyNUpHfSgcZji2gxfWj96wZg5MLgirlRXO39w49hbZ\\r\\n5E0eaGatT1jsiHrbGFg82Ui86+k3NAsjogHpRCv8d7OUXYA1YOrWqOHDZrZ4JPpUTtjoLrUBO/Zu\\r\\nZbyb9putxys9UURR8WXRteUjG8X/g3u9VOhey5QcrgdB62/kH1BLAQIKAAoAAAAIAD1PfEjhndhx\\r\\n2QEAAHoEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIW5qYWUubWUudWshMTQ1OTAzNjgw\\r\\nMCExNDU5MTIzMTk5LnhtbFBLBQYAAAAAAQABAF0AAAAmAgAAAAA=\\r\\n'" + "'UEsDBAoAAAAIAAlJfUgBY8K+2QEAAHoEAAAvAAAAZ29vZ2xlLmNvbSFuamFlLm1lLnVrITE0NTkx\\r\\nMjMyMDAhMTQ1OTIwOTU5OS54bWytVE2TnCAQve+vsOY+ou7oaoplc8ovSM4WA61DRoEC3I9/Hxzw\\r\\nY2dTlUtO4uvu1/1eo/jlfRySVzBWKPl8yNPskIBkigvZPx9+/fxxrA/JC3nAHQA/U3YlD0mCDWhl\\r\\nXDuCo5w6OmMeVaZvJR2B9Er1A6RMjRitYMiBkYqBSOUZho8jH6lhRzvpme77vizkxZp3Z2jLlHSU\\r\\nuVbITpGLc9p+QyiWplspoohK+wYGFaeqKuvMc32tD8RRhuAkL0959pRVZfFUN1WdZ3VTY7TFQ77X\\r\\nCq2hso9qPHSGXkiSn8omLx6LzHcLyBIHyW/RImvKpvGzyIUMfWZbu+1NxVoNgn20ejoPwl5gHUR5\\r\\neySRvymkI6TT1bMFKMQpv4qRGIzCIYJWdzdsfgZI+01IwEjHd7sAdkE0cySfZc2H25h/G8k7yZRZ\\r\\npjPqbdVv1WQYtEKTIi/SqknLMq0K32DFl0ymJunII0bhsMCxHbzSYfKG8SUwuyCsVlY4f3Pj2Htk\\r\\nlzd7oKm1PmG1I+rtYmD1ZCfxrqff0CIMCw7SiU7472YtuwDlYNrOqPHTZvZ4JPpSjunkLq0BOw1u\\r\\nY7yb9h9bj1d6poii4suqa8+Hd4r/B/d2qdC9ljk5XA+Mtt/IH1BLAQIKAAoAAAAIAAlJfUgBY8K+\\r\\n2QEAAHoEAAAvAAAAAAAAAAAAAAAAAAAAAABnb29nbGUuY29tIW5qYWUubWUudWshMTQ1OTEyMzIw\\r\\nMCExNDU5MjA5NTk5LnhtbFBLBQYAAAAAAQABAF0AAAAmAgAAAAA=\\r\\n'" ] }, - "execution_count": 22, + "execution_count": 86, "metadata": {}, "output_type": "execute_result" } @@ -580,7 +612,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 63, "metadata": { "collapsed": false }, @@ -588,10 +620,10 @@ { "data": { "text/plain": [ - "b\"PK\\x03\\x04\\n\\x00\\x00\\x00\\x08\\x00=O|H\\xe1\\x9d\\xd8q\\xd9\\x01\\x00\\x00z\\x04\\x00\\x00/\\x00\\x00\\x00google.com!njae.me.uk!1459036800!1459123199.xml\\xadT\\xcbr\\xa30\\x10\\xbc\\xe7+(\\xdf\\x8d\\x10\\x06\\xd6\\xdeR\\x94=\\xe5\\x0bv\\xcf\\x94,\\x06\\xac5H*I\\xe4\\xf1\\xf7+,\\xf1\\x88\\x93\\xaa\\\\\\xf6\\x84\\xe8\\x99\\xe9\\x99\\xee\\x11\\x90\\xa7\\xb7\\xa1O^\\xc0X\\xa1\\xe4\\xe3\\x0e\\xa7\\xd9.\\x01\\xc9U#d\\xf7\\xb8\\xfb\\xf3\\xfby\\x7f\\xdc%O\\xf4\\x81\\xb4\\x00\\xcd\\x99\\xf1+}H\\x12b@+\\xe3\\xea\\x01\\x1ck\\x98c\\x13\\xe6Qe\\xbaZ\\xb2\\x01h\\xa7T\\xd7C\\xca\\xd5@\\xd0\\x02\\x86\\x1c\\x18\\x98\\xe8\\xa9T\\x9e\\xa1\\x7f\\xdf7\\x033|oG=\\xd1\\xfd\\xda\\x96\\x85\\xbcX\\xf3\\xe6\\x0c\\xab\\xb9\\x92\\x8eqW\\x0b\\xd9*zqN\\xdb\\x9f\\x08\\xc5\\xd2t-E\\x0c1i_\\xc1\\xa0\\xbc\\xa8\\xaa\\xf2\\x98y\\xae\\xcf\\xf5\\x818\\xca\\x10\\r\\xc5\\x15.\\x0e\\xf91\\xabJ\\\\\\xfe\\xc8\\x0ee\\x91\\x17\\x98\\xa05\\x1e\\xf2\\xbdV\\xa8\\r\\x93]T\\xe3\\xa13tBR\\\\\\x94\\xa7\\xecP\\x1d3\\xdf- s\\x1cds\\x8b\\xe2\\xfc\\x80O'?\\x8b\\x9c\\xc9\\xd0G\\xb6\\xa5\\xdb\\xd6T\\xa2U/\\xf8{\\xad\\xc7s/\\xec\\x05\\x96A\\x94\\xb7GR\\xf9\\x97A:@:^=[\\x80B\\x9c5W1PCP8D\\xd0\\xea\\xf6\\x86M\\xcf\\x00i\\xbf\\t\\t\\x04\\xe9\\xf8ng\\xc0\\xce\\x88\\xe6\\x8e\\xe2I\\xd6t\\xb8\\x8d\\xf9\\xd5H\\xdeI\\xae\\xcc<\\x9dQ\\xaf\\x8b~\\xabF\\xc3\\xa1\\x16\\x9a\\xe68O\\xabSZ\\x96i\\x95\\xfb\\x06\\x0b>gr5JG}(\\x1cf8\\xb6\\x83\\x17\\xd6\\x8f\\xde\\xb0f\\x0eL.\\x08\\xab\\x95\\x15\\xce\\xdf\\xdc8\\xf6\\x16\\xd9\\xe4M\\x1ehf\\xadOX\\xec\\x88z\\xdb\\x18X<\\xd9H\\xbc\\xeb\\xe974\\x0b#\\xa2\\x01\\xe9D+\\xfcw\\xb3\\x94]\\x805`\\xea\\xd6\\xa8\\xe1\\xc3f\\xb6x$\\xfaTN\\xd8\\xe8.\\xb5\\x01;\\xf6ne\\xbc\\x9b\\xf6\\x9b\\xad\\xc7+=QDQ\\xf1e\\xd1\\xb5\\xe5#\\x1b\\xc5\\xff\\x83{\\xbdT\\xe8^\\xcb\\x94\\x1c\\xae\\x07A\\xebo\\xe4\\x1fPK\\x01\\x02\\n\\x00\\n\\x00\\x00\\x00\\x08\\x00=O|H\\xe1\\x9d\\xd8q\\xd9\\x01\\x00\\x00z\\x04\\x00\\x00/\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00google.com!njae.me.uk!1459036800!1459123199.xmlPK\\x05\\x06\\x00\\x00\\x00\\x00\\x01\\x00\\x01\\x00]\\x00\\x00\\x00&\\x02\\x00\\x00\\x00\\x00\"" + "b'PK\\x03\\x04\\n\\x00\\x00\\x00\\x08\\x00\\tI}H\\x01c\\xc2\\xbe\\xd9\\x01\\x00\\x00z\\x04\\x00\\x00/\\x00\\x00\\x00google.com!njae.me.uk!1459123200!1459209599.xml\\xadTM\\x93\\x9c \\x10\\xbd\\xef\\xaf\\xb0\\xe6>\\xa2\\xee\\xe8j\\x8aes\\xca/H\\xce\\x16\\x03\\xadCF\\x81\\x02\\xdc\\x8f\\x7f\\x1f\\x1c\\xf0cgS\\x95KN\\xe2\\xeb\\xee\\xd7\\xfd^\\xa3\\xf8\\xe5}\\x1c\\x92W0V(\\xf9|\\xc8\\xd3\\xec\\x90\\x80d\\x8a\\x0b\\xd9?\\x1f~\\xfd\\xfcq\\xac\\x0f\\xc9\\x0by\\xc0\\x1d\\x00?Sv%\\x0fI\\x82\\rhe\\\\;\\x82\\xa3\\x9c::c\\x1eU\\xa6o%\\x1d\\x81\\xf4J\\xf5\\x03\\xa4L\\x8d\\x18\\xad`\\xc8\\x81\\x91\\x8a\\x81H\\xe5\\x19\\x86\\x8f#\\x1f\\xa9aG;\\xe9\\x99\\xee\\xfb\\xbe,\\xe4\\xc5\\x9awgh\\xcb\\x94t\\x94\\xb9V\\xc8N\\x91\\x8bs\\xda~C(\\x96\\xa6[)\\xa2\\x88J\\xfb\\x06\\x06\\x15\\xa7\\xaa*\\xeb\\xccs}\\xad\\x0f\\xc4Q\\x86\\xe0$/Oy\\xf6\\x94Ue\\xf1T7U\\x9dguSc\\xb4\\xc5C\\xbe\\xd7\\n\\xad\\xa1\\xb2\\x8fja\\xb5#\\xea\\xedb`\\xf5d\\'\\xf1\\xae\\xa7\\xdf\\xd0\"\\x0c\\x0b\\x0e\\xd2\\x89N\\xf8\\xeff-\\xbb\\x00\\xe5`\\xda\\xce\\xa8\\xf1\\xd3f\\xf6x$\\xfaR\\x8e\\xe9\\xe4.\\xad\\x01;\\rnc\\xbc\\x9b\\xf6\\x1f[\\x8fWz\\xa6\\x88\\xa2\\xe2\\xcb\\xaak\\xcf\\x87w\\x8a\\xff\\x07\\xf7v\\xa9\\xd0\\xbd\\x9699\\\\\\x0f\\x8c\\xb6\\xdf\\xc8\\x1fPK\\x01\\x02\\n\\x00\\n\\x00\\x00\\x00\\x08\\x00\\tI}H\\x01c\\xc2\\xbe\\xd9\\x01\\x00\\x00z\\x04\\x00\\x00/\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00google.com!njae.me.uk!1459123200!1459209599.xmlPK\\x05\\x06\\x00\\x00\\x00\\x00\\x01\\x00\\x01\\x00]\\x00\\x00\\x00&\\x02\\x00\\x00\\x00\\x00'" ] }, - "execution_count": 23, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -602,7 +634,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 64, "metadata": { "collapsed": true }, @@ -614,7 +646,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 65, "metadata": { "collapsed": false }, @@ -622,10 +654,10 @@ { "data": { "text/plain": [ - "'google.com!njae.me.uk!1459036800!1459123199.xml'" + "'google.com!njae.me.uk!1459123200!1459209599.xml'" ] }, - "execution_count": 25, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } @@ -636,7 +668,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 66, "metadata": { "collapsed": true }, @@ -650,7 +682,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 67, "metadata": { "collapsed": true }, @@ -661,7 +693,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 68, "metadata": { "collapsed": false }, @@ -674,10 +706,10 @@ " org_name : google.com\n", " email : noreply-dmarc-support@google.com\n", " extra_contact_info : https://support.google.com/a/answer/2466580\n", - " report_id : 16143280651570354241\n", + " report_id : 15410706527896810898\n", " date_range : \n", - " begin : 1459036800\n", - " end : 1459123199\n", + " begin : 1459123200\n", + " end : 1459209599\n", " policy_published : \n", " domain : njae.me.uk\n", " adkim : r\n", @@ -688,7 +720,7 @@ " record : \n", " row : \n", " source_ip : 212.69.55.62\n", - " count : 2\n", + " count : 3\n", " policy_evaluated : \n", " disposition : none\n", " dkim : pass\n", @@ -715,30 +747,30 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 88, "metadata": { "collapsed": false }, "outputs": [ { - "ename": "RuntimeError", - "evalue": "No active exception to reraise", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mRuntimeError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mraise\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[1;31mRuntimeError\u001b[0m: No active exception to reraise" - ] + "data": { + "text/plain": [ + "b'Return-Path: \\r\\nDelivered-To: neil@njae.me.uk\\r\\nReceived: from localhost (localhost [127.0.0.1])\\r\\n\\tby mail.njae.me.uk (Postfix) with ESMTP id 2F3B2B4\\r\\n\\tfor ; Mon, 23 Nov 2015 16:53:20 +0000 (GMT)\\r\\nX-Virus-Scanned: Debian amavisd-new at njae.me.uk\\r\\nAuthentication-Results: ogedei.njae.me.uk (amavisd-new);\\r\\n\\tdkim=pass (2048-bit key) header.d=yahoo.com\\r\\nReceived: from mail.njae.me.uk ([127.0.0.1])\\r\\n\\tby localhost (ogedei.njae.me.uk [127.0.0.1]) (amavisd-new, port 10024)\\r\\n\\twith ESMTP id phn6BeTXjy5q for ;\\r\\n\\tMon, 23 Nov 2015 16:53:17 +0000 (GMT)\\r\\nReceived-SPF: None (no SPF record) identity=mailfrom; client-ip=98.138.229.247; helo=n4-vm7.bullet.mail.ne1.yahoo.com; envelope-from=noreply@dmarc.yahoo.com; receiver=dmarc@njae.me.uk \\r\\nAuthentication-Results: mail.njae.me.uk; dkim=pass\\r\\n\\treason=\"2048-bit key; unprotected key\"\\r\\n\\theader.d=yahoo.com header.i=@yahoo.com header.b=dV1Y/4Bc;\\r\\n\\tdkim-adsp=none (unprotected policy); dkim-atps=neutral\\r\\nReceived: from n4-vm7.bullet.mail.ne1.yahoo.com (n4-vm7.bullet.mail.ne1.yahoo.com [98.138.229.247])\\r\\n\\tby mail.njae.me.uk (Postfix) with ESMTPS id A0EA23F\\r\\n\\tfor ; Mon, 23 Nov 2015 16:53:16 +0000 (GMT)\\r\\nDKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=yahoo.com; s=s2048; t=1448297595; bh=oUT0dgwPXhYXV7MKOeA6qc/HcuG/XlY+/W2VclJfB2U=; h=Date:From:To:Subject:From:Subject; b=dV1Y/4Bc0OUvcxyeQZb7hvNYcMSim3elky+sVqi9TMTXDnOOvrzUhZYuz2FMfbQefAz5720bX7vuztbVJAcSWMw3AgNBWfSLyJ2hCLFCipY+R7uqDBA6uAz9WY0jfVVUsyr8Naz2+0J1bK6DAF76AQnfNW0I/2uWjOncYa6YsHjAHs/iOhimhzn0DBrHYW5bU99OSndh7ixjywLryJobMInXWdB8pBXxji6H6C3+80Bn1ZoEGWco5Dwr7W/7LlWlURFxLybxuOBLfAi9ihosOeIUyEpycTOz16VSdG/lm5ycxStdlmw6AXc8GkR6jHkOAZQbm6W3vlFNzvUQbtRWQg==\\r\\nReceived: from [98.138.101.135] by n4.bullet.mail.ne1.yahoo.com with NNFMP; 23 Nov 2015 16:53:15 -0000\\r\\nReceived: from [98.138.237.130] by t7.bullet.mail.ne1.yahoo.com with NNFMP; 23 Nov 2015 16:53:15 -0000\\r\\nReceived: from [127.0.0.1] by launcher100.asd.mail.corp.ne1.yahoo.com with NNFMP; 23 Nov 2015 16:53:15 -0000\\r\\nX-Yahoo-Newman-Property: asdreport\\r\\nX-Yahoo-Newman-Id: 1448246712.259522\\r\\nMIME-Version: 1.0\\r\\nContent-Transfer-Encoding: binary\\r\\nContent-Type: multipart/mixed; boundary=\"_----------=_14482975952874280\"\\r\\nX-Mailer: MIME::Lite 3.027 (F2.73; A2.04; B3.13; Q3.13)\\r\\nDate: Mon, 23 Nov 2015 08:53:15 -0800\\r\\nFrom: noreply@dmarc.yahoo.com\\r\\nTo: dmarc@njae.me.uk\\t\\r\\nSubject: Report Domain: njae.me.uk Submitter: yahoo.com Report-ID: <1448246712.259522>\\r\\nMessage-Id: <1448297594.757634@dmarc.yahoo.com>\\r\\n\\r\\n This is a multi-part message in MIME format.\\r\\n\\r\\n--_----------=_14482975952874280\\r\\nContent-Disposition: inline\\r\\nContent-Length: 44\\r\\nContent-Transfer-Encoding: binary\\r\\nContent-Type: text/plain\\r\\n\\r\\nThis is an aggregate report from Yahoo! Inc.\\r\\n--_----------=_14482975952874280\\r\\nContent-Disposition: attachment; filename=\"yahoo.com!njae.me.uk!1448150400!1448236799.zip\"\\r\\nContent-Transfer-Encoding: base64\\r\\nContent-Type: application/x-zip-compressed; name=\"yahoo.com!njae.me.uk!1448150400!1448236799.zip\"\\r\\n\\r\\nUEsDBBQAAAAIAKdGd0dVgJtgpQEAACMEAAAuABwAeWFob28uY29tIW5qYWUu\\r\\nbWUudWshMTQ0ODE1MDQwMCExNDQ4MjM2Nzk5LnhtbFVUCQADekRTVnpEU1Z1\\r\\neAsAAQQAAAAABAAAAACVU8tyozAQPGe/guQDJGBtJ67SyrnuJ+yJksVga40e\\r\\nJYns5u8zGFlgO1WpnBiaVk/3aGC7/7ov3sAHZc2vp4qUTzv+8IN1AO1eyBPW\\r\\nRcE8OOtjoyGKVkRxBhG2/tAYoYH/EUdrH4vfRhJGM5pYoIXqubMhahEi+NdW\\r\\nCy/J+3iGSKsZnRiJnnqpller1Uu92jxXNanX23VdMzp/TGy0A40X5nBph9ge\\r\\nDsqcT1frclWWjE5IJoBJ4j83z9ttgQZMFqQ3irnnVXjmbK/ke+OGfa/CEWY/\\r\\nFrMYbv4KIBrIcELBCUoE0Z6U5p7RqbigwXVncHwmzHFjDTDqMiAjr8Y4YzF5\\r\\n+9QGjlBany15+2+OHuzgJTTK8c2a1CUpSYVjndFMlHYw2I7Rqch46ghvoh9w\\r\\nUm3+MoZXAa9ZRVyl5H2JLIljdCdCQMY8hckgDqDDdUBTeRSLpLd98XpyPKZa\\r\\nMFF1Crd5PnkE0YJvOm/11bUs8YvWvQATQzw2HsLQx4Xoreuvrj1t9qiScqeX\\r\\nOeCVJFuG/6Z+sF2cJnjXY7Fd9C7YyE97w+ji//8AUEsBAh4DFAAAAAgAp0Z3\\r\\nR1WAm2ClAQAAIwQAAC4AGAAAAAAAAQAAAKSBAAAAAHlhaG9vLmNvbSFuamFl\\r\\nLm1lLnVrITE0NDgxNTA0MDAhMTQ0ODIzNjc5OS54bWxVVAUAA3pEU1Z1eAsA\\r\\nAQQAAAAABAAAAABQSwUGAAAAAAEAAQB0AAAADQIAAAAA\\r\\n\\r\\n--_----------=_14482975952874280--\\r\\n'" + ] + }, + "execution_count": 88, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "raise" + "msg1 = mailbox.fetch(b'1', '(RFC822)')[1][0][1]\n", + "msg1" ] }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 89, "metadata": { "collapsed": false }, @@ -746,21 +778,191 @@ { "data": { "text/plain": [ - "('OK', [b'Close completed.'])" + "'multipart/mixed'" ] }, - "execution_count": 29, + "execution_count": 89, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "mailbox.close()" + "pmsg1 = email.message_from_bytes(msg1)\n", + "pmsg1.get_content_type()" + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "None\n", + "None\n", + "yahoo.com!njae.me.uk!1448150400!1448236799.zip\n" + ] + } + ], + "source": [ + "for part in pmsg1.walk():\n", + " print(part.get_filename())" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This is an aggregate report from Yahoo! Inc. text/plain\n", + "UEsDBBQAAAAIAKdGd0dVgJtgpQEAACMEAAAuABwAeWFob28uY29tIW5qYWUu\r\n", + "bWUudWshMTQ0ODE1MDQwMCExNDQ4MjM2Nzk5LnhtbFVUCQADekRTVnpEU1Z1\r\n", + "eAsAAQQAAAAABAAAAACVU8tyozAQPGe/guQDJGBtJ67SyrnuJ+yJksVga40e\r\n", + "JYns5u8zGFlgO1WpnBiaVk/3aGC7/7ov3sAHZc2vp4qUTzv+8IN1AO1eyBPW\r\n", + "RcE8OOtjoyGKVkRxBhG2/tAYoYH/EUdrH4vfRhJGM5pYoIXqubMhahEi+NdW\r\n", + "Cy/J+3iGSKsZnRiJnnqpller1Uu92jxXNanX23VdMzp/TGy0A40X5nBph9ge\r\n", + "DsqcT1frclWWjE5IJoBJ4j83z9ttgQZMFqQ3irnnVXjmbK/ke+OGfa/CEWY/\r\n", + "FrMYbv4KIBrIcELBCUoE0Z6U5p7RqbigwXVncHwmzHFjDTDqMiAjr8Y4YzF5\r\n", + "+9QGjlBany15+2+OHuzgJTTK8c2a1CUpSYVjndFMlHYw2I7Rqch46ghvoh9w\r\n", + "Um3+MoZXAa9ZRVyl5H2JLIljdCdCQMY8hckgDqDDdUBTeRSLpLd98XpyPKZa\r\n", + "MFF1Crd5PnkE0YJvOm/11bUs8YvWvQATQzw2HsLQx4Xoreuvrj1t9qiScqeX\r\n", + "OeCVJFuG/6Z+sF2cJnjXY7Fd9C7YyE97w+ji//8AUEsBAh4DFAAAAAgAp0Z3\r\n", + "R1WAm2ClAQAAIwQAAC4AGAAAAAAAAQAAAKSBAAAAAHlhaG9vLmNvbSFuamFl\r\n", + "Lm1lLnVrITE0NDgxNTA0MDAhMTQ0ODIzNjc5OS54bWxVVAUAA3pEU1Z1eAsA\r\n", + "AQQAAAAABAAAAABQSwUGAAAAAAEAAQB0AAAADQIAAAAA\r\n", + " application/x-zip-compressed\n" + ] + } + ], + "source": [ + "for p in pmsg1.get_payload():\n", + " print(p.get_payload(), p.get_content_type())" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "application/x-zip-compressed \n" + ] + } + ], + "source": [ + "for p in pmsg1.get_payload():\n", + " if 'zip' in p.get_content_type():\n", + " print(p.get_content_type(), zipfile.ZipFile(io.BytesIO(p.get_payload(decode=True))))" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "application/x-zip-compressed\n", + " report_metadata : \n", + " org_name : Yahoo! Inc.\n", + " email : postmaster@dmarc.yahoo.com\n", + " report_id : 1448246712.259522\n", + " date_range : \n", + " begin : 1448150400\n", + " end : 1448236799\n", + " policy_published : \n", + " domain : njae.me.uk\n", + " adkim : r\n", + " aspf : r\n", + " p : none\n", + " pct : 100\n", + " record : \n", + " row : \n", + " source_ip : 65.20.0.12\n", + " count : 1\n", + " policy_evaluated : \n", + " disposition : none\n", + " dkim : pass\n", + " spf : fail\n", + " identifiers : \n", + " header_from : njae.me.uk\n", + " auth_results : \n", + " dkim : \n", + " domain : njae.me.uk\n", + " result : pass\n", + " spf : \n", + " domain : njae.me.uk\n", + " result : softfail\n" + ] + } + ], + "source": [ + "for p in pmsg1.get_payload():\n", + " if 'zip' in p.get_content_type():\n", + " print(p.get_content_type())\n", + " with zipfile.ZipFile(io.BytesIO(p.get_payload(decode=True))) as zf:\n", + " fn = zf.infolist()[0].filename\n", + " contents = zf.read(fn).decode('utf-8')\n", + " root = xml.etree.ElementTree.fromstring(contents)\n", + " walk(root)" + ] + }, + { + "cell_type": "code", + "execution_count": 105, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def xml_of_part(part):\n", + " with zipfile.ZipFile(io.BytesIO(part.get_payload(decode=True))) as zf:\n", + " fn = zf.infolist()[0].filename\n", + " contents = zf.read(fn).decode('utf-8')\n", + " return xml.etree.ElementTree.fromstring(contents)" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 98, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def xml_of(message):\n", + " reports = []\n", + " if message.is_multipart():\n", + " for p in pmsg1.get_payload():\n", + " if 'zip' in p.get_content_type():\n", + " reports += [xml_of_part(p)]\n", + " else:\n", + " reports = [xml_of_part(message)]\n", + " return reports" + ] + }, + { + "cell_type": "code", + "execution_count": 104, "metadata": { "collapsed": false }, @@ -768,14 +970,108 @@ { "data": { "text/plain": [ - "('BYE', [b'Logging out'])" + "[('google.com!njae.me.uk!1459123200!1459209599.xml',\n", + " )]" ] }, - "execution_count": 30, + "execution_count": 104, "metadata": {}, "output_type": "execute_result" } ], + "source": [ + "xml_of(pmsg)" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " report_metadata : \n", + " org_name : Yahoo! Inc.\n", + " email : postmaster@dmarc.yahoo.com\n", + " report_id : 1448246712.259522\n", + " date_range : \n", + " begin : 1448150400\n", + " end : 1448236799\n", + " policy_published : \n", + " domain : njae.me.uk\n", + " adkim : r\n", + " aspf : r\n", + " p : none\n", + " pct : 100\n", + " record : \n", + " row : \n", + " source_ip : 65.20.0.12\n", + " count : 1\n", + " policy_evaluated : \n", + " disposition : none\n", + " dkim : pass\n", + " spf : fail\n", + " identifiers : \n", + " header_from : njae.me.uk\n", + " auth_results : \n", + " dkim : \n", + " domain : njae.me.uk\n", + " result : pass\n", + " spf : \n", + " domain : njae.me.uk\n", + " result : softfail\n" + ] + } + ], + "source": [ + "walk(xml_of(pmsg1)[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "RuntimeError", + "evalue": "No active exception to reraise", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mRuntimeError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mraise\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;31mRuntimeError\u001b[0m: No active exception to reraise" + ] + } + ], + "source": [ + "raise" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "mailbox.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], "source": [ "mailbox.logout()" ] diff --git a/write-to-database.ipynb b/write-to-database.ipynb index 96884af..61cab59 100644 --- a/write-to-database.ipynb +++ b/write-to-database.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 92, + "execution_count": 1, "metadata": { "collapsed": true }, @@ -40,7 +40,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": { "collapsed": false }, @@ -51,7 +51,7 @@ "['imap', 'database']" ] }, - "execution_count": 4, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -62,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "metadata": { "collapsed": false }, @@ -73,7 +73,7 @@ "['server', 'username', 'port', 'password']" ] }, - "execution_count": 6, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -84,7 +84,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "metadata": { "collapsed": false }, @@ -95,7 +95,7 @@ "['server', 'database', 'username', 'password']" ] }, - "execution_count": 7, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -106,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 6, "metadata": { "collapsed": true }, @@ -124,7 +124,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 7, "metadata": { "collapsed": false }, @@ -144,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 8, "metadata": { "collapsed": false }, @@ -159,7 +159,7 @@ " 'yahoo.com!njae.me.uk!1459036800!1459123199.xml']" ] }, - "execution_count": 17, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -171,7 +171,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 9, "metadata": { "collapsed": false }, @@ -179,14 +179,14 @@ { "data": { "text/plain": [ - "{'google.com!njae.me.uk!1458777600!1458863999.xml': ,\n", - " 'google.com!njae.me.uk!1458950400!1459036799.xml': ,\n", - " 'hotmail.com!njae.me.uk!1459011600!1459098000.xml': ,\n", - " 'yahoo.com!njae.me.uk!1458864000!1458950399.xml': ,\n", - " 'yahoo.com!njae.me.uk!1459036800!1459123199.xml': }" + "{'google.com!njae.me.uk!1458777600!1458863999.xml': ,\n", + " 'google.com!njae.me.uk!1458950400!1459036799.xml': ,\n", + " 'hotmail.com!njae.me.uk!1459011600!1459098000.xml': ,\n", + " 'yahoo.com!njae.me.uk!1458864000!1458950399.xml': ,\n", + " 'yahoo.com!njae.me.uk!1459036800!1459123199.xml': }" ] }, - "execution_count": 19, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -198,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 10, "metadata": { "collapsed": false }, @@ -211,10 +211,10 @@ " org_name : google.com\n", " email : noreply-dmarc-support@google.com\n", " extra_contact_info : https://support.google.com/a/answer/2466580\n", - " report_id : 2150510829392606201\n", + " report_id : 13032699446207263761\n", " date_range : \n", - " begin : 1458950400\n", - " end : 1459036799\n", + " begin : 1458777600\n", + " end : 1458863999\n", " policy_published : \n", " domain : njae.me.uk\n", " adkim : r\n", @@ -224,29 +224,12 @@ " pct : 100\n", " record : \n", " row : \n", - " source_ip : 82.109.184.9\n", - " count : 1\n", - " policy_evaluated : \n", - " disposition : none\n", - " dkim : fail\n", - " spf : fail\n", - " identifiers : \n", - " header_from : njae.me.uk\n", - " auth_results : \n", - " dkim : \n", - " domain : clublloyds.com\n", - " result : pass\n", - " spf : \n", - " domain : clublloyds.com\n", - " result : pass\n", - " record : \n", - " row : \n", - " source_ip : 212.69.55.62\n", + " source_ip : 146.185.136.235\n", " count : 1\n", " policy_evaluated : \n", " disposition : none\n", " dkim : pass\n", - " spf : pass\n", + " spf : fail\n", " identifiers : \n", " header_from : njae.me.uk\n", " auth_results : \n", @@ -255,7 +238,7 @@ " result : pass\n", " spf : \n", " domain : njae.me.uk\n", - " result : pass\n", + " result : softfail\n", "\n", " report_metadata : \n", " org_name : Yahoo! Inc.\n", @@ -289,22 +272,41 @@ " result : softfail\n", "\n", " report_metadata : \n", - " org_name : Yahoo! Inc.\n", - " email : postmaster@dmarc.yahoo.com\n", - " report_id : 1458957186.548175\n", + " org_name : google.com\n", + " email : noreply-dmarc-support@google.com\n", + " extra_contact_info : https://support.google.com/a/answer/2466580\n", + " report_id : 2150510829392606201\n", " date_range : \n", - " begin : 1458864000\n", - " end : 1458950399\n", + " begin : 1458950400\n", + " end : 1459036799\n", " policy_published : \n", " domain : njae.me.uk\n", " adkim : r\n", " aspf : r\n", " p : none\n", + " sp : none\n", " pct : 100\n", " record : \n", " row : \n", + " source_ip : 82.109.184.9\n", + " count : 1\n", + " policy_evaluated : \n", + " disposition : none\n", + " dkim : fail\n", + " spf : fail\n", + " identifiers : \n", + " header_from : njae.me.uk\n", + " auth_results : \n", + " dkim : \n", + " domain : clublloyds.com\n", + " result : pass\n", + " spf : \n", + " domain : clublloyds.com\n", + " result : pass\n", + " record : \n", + " row : \n", " source_ip : 212.69.55.62\n", - " count : 2\n", + " count : 1\n", " policy_evaluated : \n", " disposition : none\n", " dkim : pass\n", @@ -352,28 +354,26 @@ " result : pass\n", "\n", " report_metadata : \n", - " org_name : google.com\n", - " email : noreply-dmarc-support@google.com\n", - " extra_contact_info : https://support.google.com/a/answer/2466580\n", - " report_id : 13032699446207263761\n", + " org_name : Yahoo! Inc.\n", + " email : postmaster@dmarc.yahoo.com\n", + " report_id : 1458957186.548175\n", " date_range : \n", - " begin : 1458777600\n", - " end : 1458863999\n", + " begin : 1458864000\n", + " end : 1458950399\n", " policy_published : \n", " domain : njae.me.uk\n", " adkim : r\n", " aspf : r\n", " p : none\n", - " sp : none\n", " pct : 100\n", " record : \n", " row : \n", - " source_ip : 146.185.136.235\n", - " count : 1\n", + " source_ip : 212.69.55.62\n", + " count : 2\n", " policy_evaluated : \n", " disposition : none\n", " dkim : pass\n", - " spf : fail\n", + " spf : pass\n", " identifiers : \n", " header_from : njae.me.uk\n", " auth_results : \n", @@ -382,7 +382,7 @@ " result : pass\n", " spf : \n", " domain : njae.me.uk\n", - " result : softfail\n", + " result : pass\n", "\n" ] } @@ -395,7 +395,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 11, "metadata": { "collapsed": false }, @@ -444,7 +444,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 12, "metadata": { "collapsed": false }, @@ -458,7 +458,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 13, "metadata": { "collapsed": false }, @@ -466,10 +466,10 @@ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 30, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -481,7 +481,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 14, "metadata": { "collapsed": false }, @@ -489,10 +489,14 @@ { "data": { "text/plain": [ - "{'email': 'varchar', 'org_name': 'varchar', 'report_id': 'varchar'}" + "{'date_range_begin': 'timestamp',\n", + " 'date_range_end': 'timestamp',\n", + " 'email': 'varchar',\n", + " 'org_name': 'varchar',\n", + " 'report_id': 'varchar'}" ] }, - "execution_count": 31, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -509,7 +513,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 15, "metadata": { "collapsed": false }, @@ -532,7 +536,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 16, "metadata": { "collapsed": false }, @@ -562,7 +566,7 @@ " 'feedback/report_metadata/report_id': '1459129809.695034'}" ] }, - "execution_count": 66, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -573,7 +577,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 17, "metadata": { "collapsed": false }, @@ -584,7 +588,7 @@ "'1459036800'" ] }, - "execution_count": 63, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -595,7 +599,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 18, "metadata": { "collapsed": false }, @@ -606,7 +610,7 @@ "'./report_metadata/date_range/begin'" ] }, - "execution_count": 77, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -617,7 +621,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 19, "metadata": { "collapsed": false }, @@ -628,7 +632,7 @@ "'report_metadata_date_range_begin'" ] }, - "execution_count": 82, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -640,7 +644,7 @@ }, { "cell_type": "code", - "execution_count": 156, + "execution_count": 20, "metadata": { "collapsed": false, "scrolled": true @@ -711,7 +715,7 @@ " 'pg_type': 'varchar'}}" ] }, - "execution_count": 156, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" }