MongoDB + Arctic

In [1]:

! pip3 install git+https://github.com/manahl/arctic.git

Collecting git+https://github.com/manahl/arctic.git
  Cloning https://github.com/manahl/arctic.git to /tmp/pip-p05wy4w3-build
  Requirement already satisfied (use --upgrade to upgrade): arctic==1.59.0 from git+https://github.com/manahl/arctic.git in ./.local/lib/python3.5/site-packages
Requirement already satisfied: cython in /usr/local/lib/python3.5/dist-packages (from arctic==1.59.0)
Requirement already satisfied: decorator in /usr/local/lib/python3.5/dist-packages (from arctic==1.59.0)
Requirement already satisfied: enum34 in /usr/local/lib/python3.5/dist-packages (from arctic==1.59.0)
Requirement already satisfied: mockextras in ./.local/lib/python3.5/site-packages (from arctic==1.59.0)
Requirement already satisfied: pandas in /usr/local/lib/python3.5/dist-packages (from arctic==1.59.0)
Requirement already satisfied: pymongo in /usr/local/lib/python3.5/dist-packages (from arctic==1.59.0)
Requirement already satisfied: python-dateutil in /usr/local/lib/python3.5/dist-packages (from arctic==1.59.0)
Requirement already satisfied: pytz in /usr/local/lib/python3.5/dist-packages (from arctic==1.59.0)
Requirement already satisfied: tzlocal in ./.local/lib/python3.5/site-packages (from arctic==1.59.0)
Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.5/dist-packages (from pandas->arctic==1.59.0)
Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.5/dist-packages (from python-dateutil->arctic==1.59.0)

In [2]:

import arctic
arctic

<module 'arctic' from '/home/user/.local/lib/python3.5/site-packages/arctic/__init__.py'>

In [7]:

from arctic import Arctic
import quandl
import pandas as pd

# Connect to Local MONGODB
store = Arctic('localhost')

# Create the library - defaults to VersionStore
# store.initialize_library('NASDAQ')
store.initialize_library('test')

# Access the library
# library = store['NASDAQ']
testlib = store['test']

# Load some data - maybe from Quandl
# aapl = quandl.get("WIKI/AAPL", authtoken="your token here")
testdata = pd.DataFrame({
        'a': [1,2,3,4,5],
        'b': ['a', 'a', 'b', 'a', 'c']
    })

# Store the data in the library
# library.write('AAPL', aapl, metadata={'source': 'Quandl'})
testlib.write('data1', testdata, metadata={'source': 'completely made up'})

# Reading the data
#item = library.read('AAPL')
item = testlib.read('data1')
aapl = item.data
metadata = item.metadata

Library created, but couldn't enable sharding: no such command: 'enablesharding', bad cmd: '{ enablesharding: "arctic", lsid: { id: UUID("09bbf211-e9ae-4694-a188-b61db5373fb5") }, $readPreference: { mode: "secondaryPreferred" }, $db: "admin" }'. This is OK if you're not 'admin'

In [8]:

item

VersionedItem(symbol=data1,library=arctic.test,data=<class 'pandas.core.frame.DataFrame'>,version=1,metadata={'source': 'completely made up'}

In [9]:

aapl

	a	b
index
0	1	a
1	2	a
2	3	b
3	4	a
4	5	c

In [10]:

metadata

{'source': 'completely made up'}

In [13]:

! ls -lh mongodata

total 105K
-rw------- 1 user user   45 Jan 23 09:41 WiredTiger
-rw------- 1 user user   21 Jan 23 09:41 WiredTiger.lock
-rw------- 1 user user 1.1K Jan 23 10:11 WiredTiger.turtle
-rw------- 1 user user 140K Jan 23 10:11 WiredTiger.wt
-rw------- 1 user user 4.0K Jan 23 09:41 WiredTigerLAS.wt
-rw------- 1 user user  36K Jan 23 10:11 _mdb_catalog.wt
-rw------- 1 user user  16K Jan 23 09:42 collection-0-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 collection-11-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 collection-15-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 collection-18-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 09:42 collection-2-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 09:47 collection-23-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 collection-26-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 collection-28-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 10:10 collection-30-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 collection-33-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 collection-37-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 09:42 collection-4-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 collection-40-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 collection-6-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 collection-8-4249850710114014683.wt
drwx------ 2 user user    4 Jan 23 10:11 diagnostic.data
-rw------- 1 user user  16K Jan 23 09:42 index-1-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-10-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-12-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-13-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-14-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-16-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-17-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-19-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-20-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-21-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-22-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 09:47 index-24-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 09:47 index-25-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-27-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-29-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 09:42 index-3-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 10:10 index-31-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 10:10 index-32-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-34-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-35-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-36-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-38-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-39-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-41-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-42-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-43-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 10:11 index-44-4249850710114014683.wt
-rw------- 1 user user  16K Jan 23 09:42 index-5-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-7-4249850710114014683.wt
-rw------- 1 user user 4.0K Jan 23 09:42 index-9-4249850710114014683.wt
drwx------ 2 user user    5 Jan 23 09:41 journal
-rw------- 1 user user    4 Jan 23 09:41 mongod.lock
-rw------- 1 user user  36K Jan 23 10:11 sizeStorer.wt
-rw------- 1 user user  114 Jan 23 09:41 storage.bson

In [0]: