An Introduction to Numpy and Pandas

import numpy as np
import pandas as pd

import pandas as pd
s = pd.Series([1,3,5,np.nan,6,8])
dates = pd.date_range('20130101', periods=6)
df1 = pd.DataFrame(np.random.randn(6,4), index=dates, columns=list('ABCD'))
df1

df2 = pd.DataFrame({ 'E' : 1.,
   ....:                      'F' : pd.Timestamp('20130102'),
   ....:                      'G' : pd.Series(1,index=list(range(4)),dtype='float32'),
   ....:                      'H' : np.array([3] * 4,dtype='int32'),
   ....:                      'I' : pd.Categorical(["test","train","test","train"]),
   ....:                      'J' : 'foo' })
df2

tmpdf2 = df2.copy(deep=True)
tmpdf2.set_index(tmpdf2.F, inplace=True)
df3 = df1.join(tmpdf2, how='outer')
#or: df3 = pd.merge(df1, tmpdf2, how='outer', left_index=True, right_index=True)
df3

df4 = pd.DataFrame({'A': ['foo', 'bar', 'foo', 'bar',
   ....:                          'foo', 'bar', 'foo', 'foo'],
   ....:                    'B': ['one', 'one', 'two', 'three',
   ....:                          'two', 'two', 'one', 'three'],
   ....:                    'C': np.random.randn(8),
   ....:                    'D': np.random.randn(8)})
df4

c = df1.columns.tolist()
c

n = df1.index.tolist()
n

#calculate sum (across rows) for each column
df1.sum(axis=0)

pd.concat([df1,df3], axis=0, sort=False).head()

arr = np.array([1,2,3])
pd.Series(arr).values

arr.dtype

df1.A.to_numpy()

n[0]

df1.loc[n[0],]

df1.loc[:,[c[0],c[1]] ]

df1.loc[n[0],c[0]]

df1[c[0]][:5]

df1.iloc[0,]

df1.iloc[:,0]

df1.iloc[0,0]

df1.A > 0

df1[df1.A > 0]

df5 = df1.copy()
df5['K'] = ['one', 'one','two','three','four','three']
df5['K']

df5[df5['K'].isin(['two','four'])]

#sort
df1.sort_values([c[0]], ascending=False)

df1.query('A > 0')

df1[ (df1['A'] > 0) ]

s1 = pd.Series([1,2,3,4,5,6], index=pd.date_range('20130102', periods=6))
df1['F'] = s1
df1

df5 = df1.copy()
df5[df5 > 0] = -df5
df5

df5.index.to_series().apply(pd.to_datetime)

df5.index.to_series().dt.year

df1.apply(np.cumsum)

df1.apply(lambda x: x.max() - x.min())

# string ops - vectorized
df1.iloc[:,1].astype('str').str.lower()

# missing data
df1.dropna(how='any')

df1.dropna(subset=['F'], how='all')

df1.fillna(value=5)

pd.isnull(df1)

df1.append(df2, ignore_index=True, sort=True).head()

pieces = [df1[:3], df1[3:7], df1[7:]]
pd.concat(pieces).head()

df4.groupby(['A','B']).sum()

stacked = df4.stack()
stacked.index[:4]

stacked.head()

stacked.unstack().head()

df4

# pivot tables (http://pbpython.com/pandas-pivot-table-explained.html)
df4["A"] = df4["A"].astype("category")
df4["A"].cat.set_categories(["foo", "bar"],inplace=True)

pd.pivot_table(df4,index=['A','B'])	#same as:
df4.groupby(['A','B']).mean()
df4.groupby(['A','B'])['D'].agg([np.sum,np.std])

pd.pivot_table(df4,index=['A','B'], values=['D'], aggfunc=np.sum)

pd.pivot_table(df4,index=['A'], columns=['B'], values=['D'], aggfunc=np.sum)

pd.pivot_table(df4,index=['A'], columns=['B'], values=['D'], aggfunc=np.sum, fill_value=0,margins=True)

pd.pivot_table(df4, index=['A', 'B'], columns=['C'], values='D')

pd.pivot_table(df4, index=['A'], columns=['B','C'], values='D')

rng = pd.date_range('1/1/2012', periods=100, freq='S')
ts = pd.Series(np.random.randint(0, 500, len(rng)), index=rng)
ts.head()

## frequency conversion (e.g., converting secondly data into 5-minutely data)
ts.resample('5Min').sum()

## time zones
ts_utc = ts.tz_localize('UTC')
ts_utc.tz_convert('US/Eastern').head()

## format
rng = pd.date_range('1/1/2012', periods=5, freq='M')
ts = pd.Series(np.random.randn(len(rng)), index=rng)
ps = ts.to_period()
ps.to_timestamp()

## timestamp conversion
prng = pd.period_range('1990Q1', '2000Q4', freq='Q-NOV')
ts = pd.Series(np.random.randn(len(prng)), prng)
ts.index = (prng.asfreq('M', 'e') + 1).asfreq('H', 's') + 9
ts.head()

df5 = df4.copy(deep=True)

## create categories
df5['incomeranges'] = pd.cut(df5['C'], 14)
df5["A"] = df5["A"].astype('category')
df5.head()

df5.A

#rename to more meaningful
df5["A"].cat.categories = ["foo", "bar"]
df5.head()

# domain of categories
df5["A"] = df5["A"].cat.set_categories(["very bad", "bar", "medium", "foo", "very good"])
df5.sort_values(by="A").head()

df5['A'].value_counts()

## subset on category
tmp = df5[df5['A'] == "foo"]
tmp

##TODO:subset with limited column
df5[A & B][['C', 'D']][:10]

#TODO:Plotting
## options
### Make the graphs a bit prettier, and bigger
pd.set_option('display.mpl_style', 'default') 
### Always display all the columns
pd.set_option('display.line_width', 5000) 
pd.set_option('display.max_columns', 60) 
figsize(15, 5)

df2

df2.H.values

df2.H.abs().tolist()

df2.H.mean().tolist()

import sys

arr = np.random.randn(10**3)
lst = list(arr)
arr2d = arr.reshape(20,50)

print( len(lst) )
print( arr.shape )
print( arr2d.shape )

print( type(lst) )
print( type(arr) )
print( type(arr2d) )

print( type(lst[0]) )
print( arr.dtype )

#item
print( sys.getsizeof( lst[0] ) )
print( arr.itemsize )

#total
print( sys.getsizeof( lst ) )
print( sys.getsizeof( arr ) )
print( sys.getsizeof( arr2d ) )

print( arr.nbytes )
print( arr2d.nbytes )

print( arr.tobytes()[:100] )

a = np.array([1, 2, 5, 7, 8])
a[1:3] = -1
a

b = [1, 2, 5, 7, 8]
try:
    b[1:3] = -1
except:
    print("TypeError: can only assign an iterable")

a=[1,2,5,7,8]
b=a[1:5]
b[1]=3
print(a)
print(b)

a = np.array([1, 2, 5, 7, 8])
a_slice = a[1:5]
a_slice[1] = 1000
print( a )
print( a_slice)

a = np.array([1, 2, 5, 7, 8])
a_slice = a[1:5].copy()
a_slice[1] = 1000
print( a )
print( a_slice)

a = np.array([0, 1, 2, 5, 7, 8]).reshape(3,2)
a

rows_on = np.array([True, False, True])
a[rows_on,:]

(arr != 2)[:10]

arr[arr != 2] [:10]

from numpy import arange
from timeit import Timer

Nelements = 10000
Ntimeits = 10000

x = arange(Nelements)
y = range(Nelements)

t_numpy = Timer("x.sum()", "from __main__ import x")
t_list = Timer("sum(y)", "from __main__ import y")
print("numpy: %.3e" % (t_numpy.timeit(Ntimeits)/Ntimeits,))
print("list:  %.3e" % (t_list.timeit(Ntimeits)/Ntimeits,))

print( np.random.randint(1,10, 5) )
print( np.arange(1,10) )

print( np.random.rand(3,3) )

np.zeros((4,3))

np.eye(4)

arr = np.arange(1,10)
np.append(arr, 12)

print( arr.size )
print( arr.mean() )

print( arr.min() )
print( arr.max() )

print( type(arr.tolist()) )

print( arr.astype('str') )
print( arr.astype('int') )

arr = np.arange(1,10)
np.append(arr, 12)

print( arr + arr )
print( arr - arr )

#A[n+1]-A[n]
np.diff(arr, n=1)

arr * 2

arr + 4

%%timeit 
arr * 2

twos = np.ones(9) * 2

%%timeit
arr * twos

M = np.arange(1,10).reshape(3,3)
M

M.T

M.diagonal()

np.dot(M, M)

np.sum( M )

np.sum(M, axis=1)    #sum along rows, use `axis=0` for columns