X-Git-Url: https://git.njae.me.uk/?a=blobdiff_plain;f=dmarc_to_database;h=62d6a077995ba90482d13969a4aa0c08844d8e00;hb=HEAD;hp=530d548b984fa8267eb57c46cfbacdf983441c1b;hpb=9705b435f1eb6db4a872858989d8af455c1dc890;p=dmarc.git

diff --git a/dmarc_to_database b/dmarc_to_database
index 530d548..62d6a07 100755
--- a/dmarc_to_database
+++ b/dmarc_to_database
@@ -9,15 +9,28 @@ import xml.etree.ElementTree
 import psycopg2
 import re
 import datetime
+import argparse
+
+parser = argparse.ArgumentParser(description='Process DMARC records.')
+parser.add_argument('-c', '--config', action='store',
+  default='', dest='config_file',
+  help='Path to config file')
+parser.add_argument('-t', '--test', action='store_true',
+                   default=False,
+                   help='Test, but do not add records to the database')
+args = parser.parse_args()
 
 def fetch_msg(num):
     return mailbox.uid('FETCH', num, '(RFC822)')[1][0][1]
 
 def xml_of_part(part):
-    with zipfile.ZipFile(io.BytesIO(part.get_payload(decode=True))) as zf:
+    try:
+        with zipfile.ZipFile(io.BytesIO(part.get_payload(decode=True))) as zf:
             fn = zf.infolist()[0].filename
             contents = zf.read(fn).decode('utf-8')
             return xml.etree.ElementTree.fromstring(contents)
+    except zipfile.BadZipFile:
+        return None
 
 
 def xml_of(message):
@@ -117,7 +130,7 @@ def build_insert_command(table_name, report, preamble_values=None, i=None):
         field_names += [field_maps[f]['pg_field_name']]
         if field_maps[f]['pg_type'] == 'int':
             values[field_maps[f]['pg_field_name']] = int(report.find(fp).text)
-        elif field_maps[f]['pg_type'] == 'timestamp':
+        elif field_maps[f]['pg_type'] == 'timestamptz':
             # values[field_maps[f]['pg_field_name']] = datetime.datetime.utcfromtimestamp(int(report.find(fp).text))
             values[field_maps[f]['pg_field_name']] = \
                 datetime.datetime.fromtimestamp(int(report.find(fp).text),  
@@ -138,27 +151,36 @@ def build_insert_command(table_name, report, preamble_values=None, i=None):
 
 
 def write_report(connection, cursor, report):
-    insert_string, values = build_insert_command('reports', report)
-    # print(insert_string, values)
-    cursor.execute(insert_string, values)
-    
-    for i in range(1, len(report.findall('./record'))+1):
-        field_names = []
-        cursor.execute('select id, report_metadata_report_id from reports where report_metadata_report_id = %s;', 
-            [report.find('./report_metadata/report_id').text])
-        results = cursor.fetchall()
-        if len(results) != 1:
-            raise RuntimeError('Could not find report record for report item')
-        else:
-            report_id = results[0][0]
-        insert_string, values = build_insert_command('report_items', report, i=i,
-                                                     preamble_values={'report_id': report_id})
+    try:
+        insert_string, values = build_insert_command('reports', report)
         # print(insert_string, values)
         cursor.execute(insert_string, values)
-    connection.commit()
+        
+        for i in range(1, len(report.findall('./record'))+1):
+            field_names = []
+            cursor.execute('select id, report_metadata_report_id from reports where report_metadata_report_id = %s;', 
+                [report.find('./report_metadata/report_id').text])
+            results = cursor.fetchall()
+            if len(results) != 1:
+                raise RuntimeError('Could not find report record for report item')
+            else:
+                report_id = results[0][0]
+            insert_string, values = build_insert_command('report_items', report, i=i,
+                                                         preamble_values={'report_id': report_id})
+            # print(insert_string, values)
+            cursor.execute(insert_string, values)
+        connection.commit()
+    except AttributeError:
+        pass
 
 config = configparser.ConfigParser()
-config.read('dmarc.ini')
+if args.config_file:
+    config.read(args.config_file)
+else:
+    config.read(['/etc/dmarc_to_database.ini', './dmarc_to_database.ini'])
+
+if not config.sections():
+    raise RuntimeError('Could not find configuration file')
 
 conn = psycopg2.connect(host=config['database']['server'],
                         database=config['database']['database'], 
@@ -185,7 +207,8 @@ resp, nums = mailbox.uid('SEARCH', None, mails_from)
 
 
 dmarc_reports = [report for report_set in [extract_report(fetch_msg(n)) for n in nums[0].split()]
-                for report in report_set]
+                for report in report_set
+                if report]
 
 mailbox.close()
 mailbox.logout()
@@ -196,6 +219,7 @@ for report in dmarc_reports:
     results = cur.fetchall()
     if not results:
         print('write', report.find('./report_metadata/report_id').text)
-        write_report(conn, cur, report)
+        if not args.test:
+            write_report(conn, cur, report)
 
 conn.close()