{
 "metadata": {
  "name": "",
  "signature": "sha256:927ceda74473324437d8649d3779263015689fe144737598a28a54c201ea08fb"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from dockeringMongo import *"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "c = docker.Client(base_url='unix://var/run/docker.sock',\n",
      "                  version='1.10',\n",
      "                  timeout=10)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Fire up three MongoDB containers that we'll use in a replica set\n",
      "\n",
      "#STUB is an identifier used to label the nodes\n",
      "#Superstition - short stub required?\n",
      "STUB='rs3'\n",
      "rsc=rs_config(c,STUB,num=3)\n",
      "rsc"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "{'_id': 'rs3',\n",
        " 'members': [{'_id': 0, 'host': '172.17.0.2:27017'},\n",
        "  {'_id': 1, 'host': '172.17.0.3:27017'},\n",
        "  {'_id': 2, 'host': '172.17.0.4:27017'}]}"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "docker_ps(c)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "[{'Image': 'dev24/mongodb:latest',\n",
        "  'Id': '91f70992f844dfd7117e86ca7eb3b88f69071cc5ff32766bac75220f9bc291cd',\n",
        "  'Ports': [{'PublicPort': 49155,\n",
        "    'Type': 'tcp',\n",
        "    'IP': '0.0.0.0',\n",
        "    'PrivatePort': 27017}],\n",
        "  'Command': 'usr/bin/mongod --replSet rs3 --noprealloc --smallfiles',\n",
        "  'Status': 'Up 2 seconds',\n",
        "  'Names': ['/rs3_srv2'],\n",
        "  'Created': 1423754657},\n",
        " {'Image': 'dev24/mongodb:latest',\n",
        "  'Id': 'd7bfb0b8e48a8144355134c0ebcbc5b13ee1e23ced01a20384f9608a9db10ec2',\n",
        "  'Ports': [{'PublicPort': 49154,\n",
        "    'Type': 'tcp',\n",
        "    'IP': '0.0.0.0',\n",
        "    'PrivatePort': 27017}],\n",
        "  'Command': 'usr/bin/mongod --replSet rs3 --noprealloc --smallfiles',\n",
        "  'Status': 'Up 2 seconds',\n",
        "  'Names': ['/rs3_srv1'],\n",
        "  'Created': 1423754657},\n",
        " {'Image': 'dev24/mongodb:latest',\n",
        "  'Id': '91e626b350c0dda8749bb237a38ba102467cd12030d939be53cac38df59358e1',\n",
        "  'Ports': [{'PublicPort': 49153,\n",
        "    'Type': 'tcp',\n",
        "    'IP': '0.0.0.0',\n",
        "    'PrivatePort': 27017}],\n",
        "  'Command': 'usr/bin/mongod --replSet rs3 --noprealloc --smallfiles',\n",
        "  'Status': 'Up 2 seconds',\n",
        "  'Names': ['/rs3_srv0'],\n",
        "  'Created': 1423754657}]"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Initialise the replica set\n",
      "from pymongo import MongoClient, ReadPreference\n",
      "\n",
      "#We'll use the 0th server in the set as a the node\n",
      "mc = MongoClient('localhost', get27017tcp_port(c,STUB+'_srv0'))"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#In the mongo console, we would typically use the command MONGOCLIENT.config() to initialise the replica set\n",
      "#Here, we use the replSetInitiate admin command, applying it with the desired configuration\n",
      "mc.admin.command( \"replSetInitiate\",rsc);"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#The statusReport() command lets you ask a mongoDB node what it thinks the state of the world is\n",
      "statusReport(mc)\n",
      "#Keep refeshing till we have PRIMARY and SECONDARY machines in place"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "I am machine id 0 as PRIMARY\n",
        "Machine id 1 (SECONDARY) last heard from at 2015-02-12 15:25:10\n",
        "Machine id 2 (SECONDARY) last heard from at 2015-02-12 15:25:08\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Wait a few seconds, and then reconnect, now it's part of a replica set...\n",
      "mc = MongoClient('localhost', get27017tcp_port(c,STUB+'_srv0'),replicaset=STUB)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Set up connections to the other members of the replica set\n",
      "#Again, may need to wait a few seconds - keep refreshing the cell\n",
      "client1=MongoClient('localhost', get27017tcp_port(c,STUB+'_srv1'),replicaset=STUB,read_preference=ReadPreference.SECONDARY)\n",
      "client2=MongoClient('localhost', get27017tcp_port(c,STUB+'_srv2'),replicaset=STUB,read_preference=ReadPreference.SECONDARY)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#The statusReport() command lets you ask a mongoDB node what it thinks the state of the world is\n",
      "statusReport(mc)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "I am machine id 0 as PRIMARY\n",
        "Machine id 1 (SECONDARY) last heard from at 2015-02-12 15:25:16\n",
        "Machine id 2 (SECONDARY) last heard from at 2015-02-12 15:25:16\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mcdb=mc.db\n",
      "mccoll=mcdb.testcollection"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def showAll(srv):\n",
      "    for i in srv.find(): print(i)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mccoll.insert({'name':'test1'})"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 17,
       "text": [
        "ObjectId('54dcc5e3f203c61dd5fbf16c')"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "showAll(mccoll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cl1db=client1.db\n",
      "cl1coll=cl1db.testcollection\n",
      "\n",
      "cl2db=client2.db\n",
      "cl2coll=cl2db.testcollection"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "showAll(cl1coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "showAll(cl2coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "cl1coll.insert({'name':'test2'})"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 22,
       "text": [
        "ObjectId('54dcc5ecf203c61dd5fbf16d')"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "showAll(cl1coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n",
        "{'_id': ObjectId('54dcc5ecf203c61dd5fbf16d'), 'name': 'test2'}\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "showAll(cl2coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n",
        "{'_id': ObjectId('54dcc5ecf203c61dd5fbf16d'), 'name': 'test2'}\n"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#The blockade library contains functions for setting up and tearing down iptable firewall rulesets\n",
      "from blockade import *\n",
      "rsc"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 25,
       "text": [
        "{'_id': 'rs3',\n",
        " 'members': [{'_id': 0, 'host': '172.17.0.2:27017'},\n",
        "  {'_id': 1, 'host': '172.17.0.3:27017'},\n",
        "  {'_id': 2, 'host': '172.17.0.4:27017'}]}"
       ]
      }
     ],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#The machines in each partition are put into their own lists\n",
      "#partition_containers(RULESET,[ PARTITION_1_LIST, PARTITION_2_LIST ]\n",
      "partition_containers('test1w2s2', [ [netobj('172.17.0.4')],[netobj('172.17.0.2'),netobj('172.17.0.3')]])"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mccoll.insert({'name':'test3'})\n",
      "cl1coll.insert({'name':'test4'})\n",
      "cl2coll.insert({'name':'test5'})"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 27,
       "text": [
        "ObjectId('54dcc606f203c61dd5fbf170')"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "showAll(cl1coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n",
        "{'_id': ObjectId('54dcc5ecf203c61dd5fbf16d'), 'name': 'test2'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16e'), 'name': 'test3'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16f'), 'name': 'test4'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf170'), 'name': 'test5'}\n"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#The replica set connection masks what is going on...\n",
      "showAll(cl2coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n",
        "{'_id': ObjectId('54dcc5ecf203c61dd5fbf16d'), 'name': 'test2'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16e'), 'name': 'test3'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16f'), 'name': 'test4'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf170'), 'name': 'test5'}\n"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Generate clients not to the replica set\n",
      "rawclient1=MongoClient('localhost', get27017tcp_port(c,STUB+'_srv1'),read_preference = ReadPreference.SECONDARY)\n",
      "rawclient2=MongoClient('localhost', get27017tcp_port(c,STUB+'_srv2'),read_preference = ReadPreference.SECONDARY)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rawcl1db=rawclient1.db\n",
      "rawcl1coll=rawcl1db.testcollection\n",
      "showAll(rawcl1coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n",
        "{'_id': ObjectId('54dcc5ecf203c61dd5fbf16d'), 'name': 'test2'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16e'), 'name': 'test3'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16f'), 'name': 'test4'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf170'), 'name': 'test5'}\n"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rawcl2db=rawclient2.db\n",
      "rawcl2coll=rawcl2db.testcollection\n",
      "showAll(rawcl2coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n",
        "{'_id': ObjectId('54dcc5ecf203c61dd5fbf16d'), 'name': 'test2'}\n"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mccoll.insert({'name':'test6'})\n",
      "cl1coll.insert({'name':'test7'})\n",
      "cl2coll.insert({'name':'test8'})\n",
      "#Can we also define a write concern to write on a secondary?\n",
      "#http://docs.mongodb.org/manual/core/replica-set-write-concern/"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 33,
       "text": [
        "ObjectId('54dcc612f203c61dd5fbf173')"
       ]
      }
     ],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "showAll(rawcl1coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n",
        "{'_id': ObjectId('54dcc5ecf203c61dd5fbf16d'), 'name': 'test2'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16e'), 'name': 'test3'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16f'), 'name': 'test4'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf170'), 'name': 'test5'}\n",
        "{'_id': ObjectId('54dcc612f203c61dd5fbf171'), 'name': 'test6'}\n",
        "{'_id': ObjectId('54dcc612f203c61dd5fbf172'), 'name': 'test7'}\n",
        "{'_id': ObjectId('54dcc612f203c61dd5fbf173'), 'name': 'test8'}\n"
       ]
      }
     ],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#The replica set connection masks what is going on...\n",
      "showAll(cl2coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n",
        "{'_id': ObjectId('54dcc5ecf203c61dd5fbf16d'), 'name': 'test2'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16e'), 'name': 'test3'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16f'), 'name': 'test4'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf170'), 'name': 'test5'}\n",
        "{'_id': ObjectId('54dcc612f203c61dd5fbf171'), 'name': 'test6'}\n",
        "{'_id': ObjectId('54dcc612f203c61dd5fbf172'), 'name': 'test7'}\n",
        "{'_id': ObjectId('54dcc612f203c61dd5fbf173'), 'name': 'test8'}\n"
       ]
      }
     ],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#So we can read the last synched state from a machine in the wilderness\n",
      "#But can we also define a connection that allows us to write to it?\n",
      "showAll(rawcl2coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n",
        "{'_id': ObjectId('54dcc5ecf203c61dd5fbf16d'), 'name': 'test2'}\n"
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Repair the network\n",
      "clear_iptables('test1w2s2')"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 37
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Wait a second or two for synching...\n",
      "showAll(rawcl2coll)"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'_id': ObjectId('54dcc5e3f203c61dd5fbf16c'), 'name': 'test1'}\n",
        "{'_id': ObjectId('54dcc5ecf203c61dd5fbf16d'), 'name': 'test2'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16e'), 'name': 'test3'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf16f'), 'name': 'test4'}\n",
        "{'_id': ObjectId('54dcc606f203c61dd5fbf170'), 'name': 'test5'}\n",
        "{'_id': ObjectId('54dcc612f203c61dd5fbf171'), 'name': 'test6'}\n",
        "{'_id': ObjectId('54dcc612f203c61dd5fbf172'), 'name': 'test7'}\n",
        "{'_id': ObjectId('54dcc612f203c61dd5fbf173'), 'name': 'test8'}\n"
       ]
      }
     ],
     "prompt_number": 39
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!docker.io logs {STUB}_srv2 > {STUB}_srv2_log.txt\n",
      "!tail -n 35 {STUB}_srv2_log.txt"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Thu Feb 12 15:25:23.206 [repl writer worker 1] build index done.  scanned 0 total records. 0 secs\r\n",
        "Thu Feb 12 15:25:26.630 [conn5] end connection 172.17.0.2:53719 (1 connection now open)\r\n",
        "Thu Feb 12 15:25:26.630 [initandlisten] connection accepted from 172.17.0.2:53733 #8 (2 connections now open)\r\n",
        "Thu Feb 12 15:25:38.634 [conn6] end connection 172.17.0.3:55858 (1 connection now open)\r\n",
        "Thu Feb 12 15:25:38.634 [initandlisten] connection accepted from 172.17.0.3:55871 #9 (2 connections now open)\r\n",
        "Thu Feb 12 15:26:05.370 [initandlisten] connection accepted from 172.17.42.1:39031 #10 (3 connections now open)\r\n",
        "Thu Feb 12 15:26:06.640 [rsHealthPoll] DBClientCursor::init call() failed\r\n",
        "Thu Feb 12 15:26:06.640 [rsHealthPoll] replset info 172.17.0.2:27017 heartbeat failed, retrying\r\n",
        "Thu Feb 12 15:26:06.640 [rsHealthPoll] DBClientCursor::init call() failed\r\n",
        "Thu Feb 12 15:26:06.641 [rsHealthPoll] replset info 172.17.0.3:27017 heartbeat failed, retrying\r\n",
        "Thu Feb 12 15:26:07.640 [rsHealthPoll] replSet info 172.17.0.2:27017 is down (or slow to respond): \r\n",
        "Thu Feb 12 15:26:07.640 [rsHealthPoll] replSet member 172.17.0.2:27017 is now in state DOWN\r\n",
        "Thu Feb 12 15:26:07.641 [rsHealthPoll] replSet info 172.17.0.3:27017 is down (or slow to respond): \r\n",
        "Thu Feb 12 15:26:07.641 [rsHealthPoll] replSet member 172.17.0.3:27017 is now in state DOWN\r\n",
        "Thu Feb 12 15:26:09.387 [rsMgr] replSet not electing self, not all members up and we have been up less than 5 minutes\r\n",
        "Thu Feb 12 15:26:09.387 [rsMgr] replSet can't see a majority, will not try to elect self\r\n",
        "Thu Feb 12 15:26:15.641 [rsHealthPoll] replset info 172.17.0.2:27017 heartbeat failed, retrying\r\n",
        "Thu Feb 12 15:26:15.644 [rsHealthPoll] replset info 172.17.0.3:27017 heartbeat failed, retrying\r\n",
        "Thu Feb 12 15:26:21.640 [initandlisten] connection accepted from 172.17.0.3:55893 #11 (4 connections now open)\r\n",
        "Thu Feb 12 15:26:21.645 [rsHealthPoll] replset info 172.17.0.2:27017 thinks that we are down\r\n",
        "Thu Feb 12 15:26:21.645 [rsHealthPoll] replSet member 172.17.0.2:27017 is up\r\n",
        "Thu Feb 12 15:26:21.645 [rsHealthPoll] replSet member 172.17.0.2:27017 is now in state PRIMARY\r\n",
        "Thu Feb 12 15:26:21.645 [rsHealthPoll] replSet member 172.17.0.3:27017 is up\r\n",
        "Thu Feb 12 15:26:21.645 [rsHealthPoll] replSet member 172.17.0.3:27017 is now in state SECONDARY\r\n",
        "Thu Feb 12 15:26:22.643 [initandlisten] connection accepted from 172.17.0.2:53759 #12 (5 connections now open)\r\n",
        "Thu Feb 12 15:26:22.644 [conn12] end connection 172.17.0.2:53759 (4 connections now open)\r\n",
        "Thu Feb 12 15:26:22.647 [initandlisten] connection accepted from 172.17.0.2:53760 #13 (5 connections now open)\r\n",
        "Thu Feb 12 15:26:22.804 [rsBackgroundSync] Socket recv() timeout  172.17.0.2:27017\r\n",
        "Thu Feb 12 15:26:22.804 [rsBackgroundSync] SocketException: remote: 172.17.0.2:27017 error: 9001 socket exception [RECV_TIMEOUT] server [172.17.0.2:27017] \r\n",
        "Thu Feb 12 15:26:22.811 [rsBackgroundSync] replSet sync source problem: 10278 dbclient error communicating with server: 172.17.0.2:27017\r\n",
        "Thu Feb 12 15:26:22.811 [rsBackgroundSync] replSet syncing to: 172.17.0.2:27017\r\n",
        "Thu Feb 12 15:26:22.825 [conn9] end connection 172.17.0.3:55871 (4 connections now open)\r\n",
        "Thu Feb 12 15:26:22.825 [conn8] end connection 172.17.0.2:53733 (3 connections now open)\r\n",
        "Thu Feb 12 15:26:29.643 [conn11] end connection 172.17.0.3:55893 (2 connections now open)\r\n",
        "Thu Feb 12 15:26:29.643 [initandlisten] connection accepted from 172.17.0.3:55900 #14 (3 connections now open)\r\n"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!docker.io logs {STUB}_srv0 > {STUB}_srv0_log.txt\n",
      "!tail -n 35 {STUB}_srv0_log.txt"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Thu Feb 12 15:24:56.616 [rsHealthPoll] replSet member 172.17.0.4:27017 is now in state RECOVERING\r\n",
        "Thu Feb 12 15:24:57.500 [slaveTracking] build index local.slaves { _id: 1 }\r\n",
        "Thu Feb 12 15:24:57.502 [slaveTracking] build index done.  scanned 0 total records. 0.001 secs\r\n",
        "Thu Feb 12 15:24:58.162 [rsHealthPoll] replSet member 172.17.0.3:27017 is now in state SECONDARY\r\n",
        "Thu Feb 12 15:24:58.618 [rsHealthPoll] replSet member 172.17.0.4:27017 is now in state SECONDARY\r\n",
        "Thu Feb 12 15:25:12.681 [initandlisten] connection accepted from 172.17.42.1:60113 #16 (8 connections now open)\r\n",
        "Thu Feb 12 15:25:12.683 [conn1] end connection 172.17.42.1:60082 (7 connections now open)\r\n",
        "Thu Feb 12 15:25:16.018 [initandlisten] connection accepted from 172.17.42.1:60116 #17 (8 connections now open)\r\n",
        "Thu Feb 12 15:25:16.021 [initandlisten] connection accepted from 172.17.42.1:60119 #18 (9 connections now open)\r\n",
        "Thu Feb 12 15:25:23.004 [conn16] allocating new ns file /data/db/db.ns, filling with zeroes...\r\n",
        "Thu Feb 12 15:25:23.043 [FileAllocator] allocating new datafile /data/db/db.0, filling with zeroes...\r\n",
        "Thu Feb 12 15:25:23.053 [FileAllocator] done allocating datafile /data/db/db.0, size: 16MB,  took 0.01 secs\r\n",
        "Thu Feb 12 15:25:23.054 [conn16] build index db.testcollection { _id: 1 }\r\n",
        "Thu Feb 12 15:25:23.054 [conn16] build index done.  scanned 0 total records. 0 secs\r\n",
        "Thu Feb 12 15:25:24.629 [conn9] end connection 172.17.0.4:60679 (8 connections now open)\r\n",
        "Thu Feb 12 15:25:24.629 [initandlisten] connection accepted from 172.17.0.4:60698 #19 (9 connections now open)\r\n",
        "Thu Feb 12 15:25:24.630 [conn8] end connection 172.17.0.3:49006 (8 connections now open)\r\n",
        "Thu Feb 12 15:25:24.630 [initandlisten] connection accepted from 172.17.0.3:49027 #20 (10 connections now open)\r\n",
        "Thu Feb 12 15:25:54.640 [conn20] end connection 172.17.0.3:49027 (8 connections now open)\r\n",
        "Thu Feb 12 15:25:54.641 [initandlisten] connection accepted from 172.17.0.3:49032 #21 (9 connections now open)\r\n",
        "Thu Feb 12 15:25:54.641 [conn19] end connection 172.17.0.4:60698 (8 connections now open)\r\n",
        "Thu Feb 12 15:25:54.641 [initandlisten] connection accepted from 172.17.0.4:60705 #22 (10 connections now open)\r\n",
        "Thu Feb 12 15:26:02.641 [rsHealthPoll] couldn't connect to 172.17.0.4:27017: couldn't connect to server 172.17.0.4:27017\r\n",
        "Thu Feb 12 15:26:08.642 [rsHealthPoll] couldn't connect to 172.17.0.4:27017: couldn't connect to server 172.17.0.4:27017\r\n",
        "Thu Feb 12 15:26:14.642 [rsHealthPoll] replSet info 172.17.0.4:27017 is down (or slow to respond): \r\n",
        "Thu Feb 12 15:26:14.642 [rsHealthPoll] replSet member 172.17.0.4:27017 is now in state DOWN\r\n",
        "Thu Feb 12 15:26:21.645 [initandlisten] connection accepted from 172.17.0.4:60725 #23 (10 connections now open)\r\n",
        "Thu Feb 12 15:26:22.642 [rsHealthPoll] couldn't connect to 172.17.0.4:27017: couldn't connect to server 172.17.0.4:27017\r\n",
        "Thu Feb 12 15:26:22.649 [rsHealthPoll] replSet member 172.17.0.4:27017 is up\r\n",
        "Thu Feb 12 15:26:22.649 [rsHealthPoll] replSet member 172.17.0.4:27017 is now in state SECONDARY\r\n",
        "Thu Feb 12 15:26:22.811 [conn13] end connection 172.17.0.4:60683 (9 connections now open)\r\n",
        "Thu Feb 12 15:26:22.812 [initandlisten] connection accepted from 172.17.0.4:60728 #24 (11 connections now open)\r\n",
        "Thu Feb 12 15:26:22.825 [conn22] end connection 172.17.0.4:60705 (9 connections now open)\r\n",
        "Thu Feb 12 15:26:24.647 [conn21] end connection 172.17.0.3:49032 (8 connections now open)\r\n",
        "Thu Feb 12 15:26:24.647 [initandlisten] connection accepted from 172.17.0.3:49057 #25 (9 connections now open)\r\n"
       ]
      }
     ],
     "prompt_number": 41
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Shut down the containers\n",
      "tidyAwayContainers(c,['rs3_srv0','rs3_srv1','rs3_srv2'])"
     ],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": [],
     "prompt_number": 42
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {
      "activity": false
     },
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}