9 "# Import the required libraries and open the connection to Mongo\n",
11 "import collections\n",
12 "from datetime import datetime\n",
13 "import matplotlib as mpl\n",
14 "import matplotlib.pyplot as plt\n",
15 "%matplotlib inline\n",
17 "import numpy as np\n",
18 "import pandas as pd\n",
19 "import scipy.stats\n",
22 "from bson.objectid import ObjectId\n",
23 "# client = pymongo.MongoClient('mongodb://localhost:27117/')\n",
24 "# client = pymongo.MongoClient('mongodb://ogedei:27017')\n",
25 "client = pymongo.MongoClient('mongodb://localhost:27017')"
30 "execution_count": 141,
36 "# Connect to the database\n",
37 "summerofcode_db = client.summerofcode\n",
38 "imp = summerofcode_db.imp\n",
39 "icmas = summerofcode_db.icmas"
44 "execution_count": 139,
49 "output_type": "stream",
51 "14.1 Basic CRUD.ipynb\r\n",
52 "2016J_TMA01_q1-TN.ipynb\r\n",
53 "2016J_TMA01_q2-withTN.ipynb\r\n",
54 "import-data-Copy1.ipynb\r\n",
55 "import-data.ipynb\r\n",
56 "SUMMEROFCODE-Day 0 Warm up holiday prices-grades.csv\r\n",
57 "SUMMEROFCODE-Day 1 Choosing a holiday-grades.csv\r\n",
58 "SUMMEROFCODE-Day 2 Lifts-grades.csv\r\n",
59 "SUMMEROFCODE-Day 3 Door codes-grades.csv\r\n",
60 "SUMMEROFCODE-Day 4 Beach labyrinth-grades.csv\r\n",
61 "SUMMEROFCODE-Day 5 Laser display boards-grades.csv\r\n",
62 "TMA01_Question3_Solutions.ipynb\r\n"
72 "execution_count": 102,
78 "['SUMMEROFCODE-Day 0 Warm up holiday prices-grades.csv',\n",
79 " 'SUMMEROFCODE-Day 1 Choosing a holiday-grades.csv',\n",
80 " 'SUMMEROFCODE-Day 2 Lifts-grades.csv',\n",
81 " 'SUMMEROFCODE-Day 3 Door codes-grades.csv',\n",
82 " 'SUMMEROFCODE-Day 4 Beach labyrinth-grades.csv',\n",
83 " 'SUMMEROFCODE-Day 5 Laser display boards-grades.csv']"
86 "execution_count": 102,
88 "output_type": "execute_result"
98 "execution_count": 114,
104 "def update_grades():\n",
105 " for i in imp.find(modifiers={\"$snapshot\": True}):\n",
106 " imp.update_one({'_id': i['_id']},\n",
107 " {'$set': {'grade': i['Grade/10']['00'],\n",
108 " 'q1': i['Q'][' 1 /5']['00'],\n",
109 " 'q2': i['Q'][' 2 /5']['00']},\n",
110 " '$unset': {'Grade/10': '', 'Q': ''}})"
115 "execution_count": 109,
119 "def update_dates():\n",
120 " for i in imp.find(modifiers={\"$snapshot\": True}):\n",
121 " starttime = datetime.strptime(i['Started on'].strip(), '%d %b %Y %H:%M')\n",
122 " if i['Completed'] == '-':\n",
123 " imp.update_one({'_id': i['_id']},\n",
124 " {'$set': {'started': starttime},\n",
125 " '$unset': {'Started on': '', 'Completed': '', 'Time taken': ''}})\n",
127 " endtime = datetime.strptime(i['Completed'].strip(), '%d %b %Y %H:%M')\n",
128 " imp.update_one({'_id': i['_id']},\n",
129 " {'$set': {'started': starttime,\n",
130 " 'completed': endtime},\n",
131 " '$unset': {'Started on': '', 'Completed': '', 'Time taken': ''}})"
136 "execution_count": 110,
142 "def merge_imported(icma_number):\n",
143 " icmas.delete_many({'icma_number': icma_number})\n",
145 " for i in imp.find(modifiers={\"$snapshot\": True}):\n",
147 " i['icma_number'] = icma_number\n",
148 " icmas.insert_one(i)"
153 "execution_count": 111,
159 "['SUMMEROFCODE-Day 0 Warm up holiday prices-grades.csv',\n",
160 " 'SUMMEROFCODE-Day 1 Choosing a holiday-grades.csv',\n",
161 " 'SUMMEROFCODE-Day 2 Lifts-grades.csv',\n",
162 " 'SUMMEROFCODE-Day 3 Door codes-grades.csv',\n",
163 " 'SUMMEROFCODE-Day 4 Beach labyrinth-grades.csv',\n",
164 " 'SUMMEROFCODE-Day 5 Laser display boards-grades.csv']"
167 "execution_count": 111,
169 "output_type": "execute_result"
179 "execution_count": 123,
185 "<pymongo.results.DeleteResult at 0x7f3f2249fca8>"
188 "execution_count": 123,
190 "output_type": "execute_result"
194 "icmas.delete_many({})"
199 "execution_count": 132,
204 "output_type": "stream",
206 "SUMMEROFCODE-Day 0 Warm up holiday prices-grades.csv\n",
207 "2017-07-22T15:24:32.622+0100\tconnected to: localhost\n",
208 "2017-07-22T15:24:32.622+0100\tdropping: summerofcode.imp\n",
209 "2017-07-22T15:24:32.629+0100\timported 285 documents\n",
210 "SUMMEROFCODE-Day 1 Choosing a holiday-grades.csv\n",
211 "2017-07-22T15:24:32.971+0100\tconnected to: localhost\n",
212 "2017-07-22T15:24:32.971+0100\tdropping: summerofcode.imp\n",
213 "2017-07-22T15:24:32.974+0100\timported 137 documents\n",
214 "SUMMEROFCODE-Day 2 Lifts-grades.csv\n",
215 "2017-07-22T15:24:33.202+0100\tconnected to: localhost\n",
216 "2017-07-22T15:24:33.202+0100\tdropping: summerofcode.imp\n",
217 "2017-07-22T15:24:33.204+0100\timported 92 documents\n",
218 "SUMMEROFCODE-Day 3 Door codes-grades.csv\n",
219 "2017-07-22T15:24:33.394+0100\tconnected to: localhost\n",
220 "2017-07-22T15:24:33.394+0100\tdropping: summerofcode.imp\n",
221 "2017-07-22T15:24:33.396+0100\timported 77 documents\n",
222 "SUMMEROFCODE-Day 4 Beach labyrinth-grades.csv\n",
223 "2017-07-22T15:24:33.573+0100\tconnected to: localhost\n",
224 "2017-07-22T15:24:33.573+0100\tdropping: summerofcode.imp\n",
225 "2017-07-22T15:24:33.575+0100\timported 79 documents\n",
226 "SUMMEROFCODE-Day 5 Laser display boards-grades.csv\n",
227 "2017-07-22T15:24:33.753+0100\tconnected to: localhost\n",
228 "2017-07-22T15:24:33.753+0100\tdropping: summerofcode.imp\n",
229 "2017-07-22T15:24:33.755+0100\timported 57 documents\n"
234 "for csv in csvs:\n",
236 " qcsv = \"'{}'\".format(csv)\n",
237 " !mongoimport --drop --db summerofcode --collection imp --type csv --headerline --ignoreBlanks --file {qcsv}\n",
238 " imp.delete_many({'Surname': 'Overall average'})\n",
239 " update_grades()\n",
241 " merge_imported(int(csv.split()[1]))"
246 "execution_count": 134,
255 "execution_count": 134,
257 "output_type": "execute_result"
261 "icmas.find({}, ['icma']).count()"
266 "execution_count": 138,
272 "[{'_id': 3, 'count': 76},\n",
273 " {'_id': 2, 'count': 91},\n",
274 " {'_id': 5, 'count': 56},\n",
275 " {'_id': 4, 'count': 78},\n",
276 " {'_id': 1, 'count': 136},\n",
277 " {'_id': 0, 'count': 284}]"
280 "execution_count": 138,
282 "output_type": "execute_result"
286 "pipeline = [{\"$group\": {\"_id\": \"$icma_number\", \"count\": {\"$sum\": 1}}}]\n",
287 "list(icmas.aggregate(pipeline))"
292 "execution_count": null,
302 "display_name": "Python 3",
303 "language": "python",
311 "file_extension": ".py",
312 "mimetype": "text/x-python",
314 "nbconvert_exporter": "python",
315 "pygments_lexer": "ipython3",