利用python进行数据分析 使用什么软件

用于GX处理数据的python工具。一般处理数据都需要完成以下几个步骤：

与外界进行交互

准备，数据清理、修整、整合、规范化、重塑、切片切换、变形等等

转换

建模和计算

展示

Introductory examples
1.usa.gov data from bit.ly
[code]%pwd

%cd ../book_scripts

path = 'ch02/usagov_bitly_data2012-03-16-1331923249.txt'

open(path).readline()

import json
path = 'ch02/usagov_bitly_data2012-03-16-1331923249.txt'
records = [json.loads(line) for line in open(path)]

records[0]

records[0]['tz']

print(records[0]['tz'])

Counting time zones in pure Python
[code]time_zones = [rec['tz'] for rec in records]

time_zones = [rec['tz'] for rec in records if 'tz' in rec]

time_zones[:10]

def get_counts(sequence):
counts = {}
for x in sequence:
if x in counts:
counts[x] += 1
else:
counts[x] = 1
return counts

from collections import defaultdict

def get_counts2(sequence):
counts = defaultdict(int) # values will initialize to 0
for x in sequence:
counts[x] += 1
return counts

counts = get_counts(time_zones)

counts['America/New_York']

len(time_zones)

def top_counts(count_dict， n=10):
value_key_pairs = [(count， tz) for tz， count in count_dict.items()]
value_key_pairs.sort()
return value_key_pairs[-n:]

top_counts(counts)

from collections import Counter

counts = Counter(time_zones)

counts.most_common(10)

Counting time zones with pandas
[code]%matplotlib inline

from __future__ import division
from numpy.random import randn
import numpy as np
import os
import matplotlib.pyplot as plt
import pandas as pd
plt.rc('figure'， figsize=(10， 6))
np.set_printoptions(precision=4)

import json
path = 'ch02/usagov_bitly_data2012-03-16-1331923249.txt'
lines = open(path).readlines()
records = [json.loads(line) for line in lines]

from pandas import DataFrame， Series
import pandas as pd

frame = DataFrame(records)
frame

frame['tz'][:10]

tz_counts = frame['tz'].value_counts()
tz_counts[:10]

clean_tz = frame['tz'].fillna('Missing')
clean_tz[clean_tz == ''] = 'Unknown'
tz_counts = clean_tz.value_counts()
tz_counts[:10]

plt.figure(figsize=(10， 4))

tz_counts[:10].plot(kind='barh'， rot=0)

frame['a'][1]

frame['a'][50]

frame['a'][51]

results = Series([x.split()[0] for x in frame.a.dropna()])
results[:5]

results.value_counts()[:8]

cframe = frame[frame.a.notnull()]

operating_system = np.where(cframe['a'].str.contains('Windows')，
'Windows'， 'Not Windows')
operating_system[:5]

by_tz_os = cframe.groupby(['tz'， operating_system])

agg_counts = by_tz_os.size().unstack().fillna(0)
agg_counts[:10]

# Use to sort in ascending order
indexer = agg_counts.sum(1).argsort()
indexer[:10]

count_subset = agg_counts.take(indexer)[-10:]
count_subset

plt.figure()

count_subset.plot(kind='barh'， stacked=True)

plt.figure()

normed_subset = count_subset.div(count_subset.sum(1)， axis=0)
normed_subset.plot(kind='barh'， stacked=True)

MovieLens 1M data set
[code]import pandas as pd
import os
encoding = 'latin1'

upath = os.path.expanduser('ch02/movielens/users.dat')
rpath = os.path.expanduser('ch02/movielens/ratings.dat')
mpath = os.path.expanduser('ch02/movielens/movies.dat')

unames = ['user_id'， 'gender'， 'age'， 'occupation'， 'zip']
rnames = ['user_id'， 'movie_id'， 'rating'， 'timestamp']
mnames = ['movie_id'， 'title'， 'genres']

users = pd.read_csv(upath， sep='::'， header=None， names=unames， encoding=encoding)
ratings = pd.read_csv(rpath， sep='::'， header=None， names=rnames， encoding=encoding)
movies = pd.read_csv(mpath， sep='::'， header=None， names=mnames， encoding=encoding)

users[:5]

ratings[:5]

movies[:5]

ratings

data = pd.merge(pd.merge(ratings， users)， movies)
data

data.ix[0]

mean_ratings = data.pivot_table('rating'， index='title'，
columns='gender'， aggfunc='mean')
mean_ratings[:5]

ratings_by_title = data.groupby('title').size()

ratings_by_title[:5]

active_titles = ratings_by_title.index[ratings_by_title >= 250]

active_titles[:10]

mean_ratings = mean_ratings.ix[active_titles]
mean_ratings

mean_ratings = mean_ratings.rename(index={'Seven Samurai (The Magnificent Seven) (Shichinin no samurai) (1954)':
'Seven Samurai (Shichinin no samurai) (1954)'})

top_female_ratings = mean_ratings.sort_index(by='F'， ascending=False)
top_female_ratings[:10]

Measuring rating disagreement
[code]mean_ratings['diff'] = mean_ratings['M'] - mean_ratings['F']

sorted_by_diff = mean_ratings.sort_index(by='diff')
sorted_by_diff[:15]

# Reverse order of rows， take first 15 rows
sorted_by_diff[::-1][:15]

# Standard deviation of rating grouped by title
rating_std_by_title = data.groupby('title')['rating'].std()
# Filter down to active_titles
rating_std_by_title = rating_std_by_title.ix[active_titles]
# Order Series by value in descending order
rating_std_by_title.order(ascending=False)[:10]

US Baby Names 1880-2010
[code]from __future__ import division
from numpy.random import randn
import numpy as np
import matplotlib.pyplot as plt
plt.rc('figure'， figsize=(12， 5))
np.set_printoptions(precision=4)
%pwd

http://www.ssa.gov/oact/babynames/limits.html

[code]!head -n 10 ch02/names/yob1880.txt

import pandas as pd
names1880 = pd.read_csv('ch02/names/yob1880.txt'， names=['name'， 'sex'， 'births'])
names1880

names1880.groupby('sex').births.sum()

# 2010 is the last available year right now
years = range(1880， 2011)

pieces = []
columns = ['name'， 'sex'， 'births']

for year in years:
path = 'names/names/yob%d.txt' % year
frame = pd.read_csv(path， names=columns)

frame['year'] = year
pieces.append(frame)

# Concatenate everything into a single DataFrame
names = pd.concat(pieces， ignore_index=True)

total_births = names.pivot_table('births'， index='year'，
columns='sex'， aggfunc=sum)

total_births.tail()

total_births.plot(title='Total births by sex and year')

def add_prop(group):
# Integer division floors
births = group.births.astype(float)

group['prop'] = births / births.sum()
return group
names = names.groupby(['year'， 'sex']).apply(add_prop)

names

np.allclose(names.groupby(['year'， 'sex']).prop.sum()， 1)

def get_top1000(group):
return group.sort_index(by='births'， ascending=False)[:1000]
grouped = names.groupby(['year'， 'sex'])
top1000 = grouped.apply(get_top1000)

pieces = []
for year， group in names.groupby(['year'， 'sex']):
pieces.append(group.sort_index(by='births'， ascending=False)[:1000])
top1000 = pd.concat(pieces， ignore_index=True)

top1000.index = np.arange(len(top1000))

top1000

Analyzing naming trends
[code]boys = top1000[top1000.sex == 'M']
girls = top1000[top1000.sex == 'F']

total_births = top1000.pivot_table('births'， index='year'， columns='name'，
aggfunc=sum)
total_births

subset = total_births[['John'， 'Harry'， 'Mary'， 'Marilyn']]
subset.plot(subplots=True， figsize=(12， 10)， grid=False，
title="Number of births per year")

Measuring the increase in naming diversity
[code]plt.figure()

table = top1000.pivot_table('prop'， index='year'，
columns='sex'， aggfunc=sum)
table.plot(title='Sum of table1000.prop by year and sex'，
yticks=np.linspace(0， 1.2， 13)， xticks=range(1880， 2020， 10))

df = boys[boys.year == 2010]
df

prop_cumsum = df.sort_index(by='prop'， ascending=False).prop.cumsum()
prop_cumsum[:10]

prop_cumsum.values.searchsorted(0.5)

df = boys[boys.year == 1900]
in1900 = df.sort_index(by='prop'， ascending=False).prop.cumsum()
in1900.values.searchsorted(0.5) + 1

def get_quantile_count(group， q=0.5):
group = group.sort_index(by='prop'， ascending=False)
return group.prop.cumsum().values.searchsorted(q) + 1

diversity = top1000.groupby(['year'， 'sex']).apply(get_quantile_count)
diversity = diversity.unstack('sex')

def get_quantile_count(group， q=0.5):
group = group.sort_index(by='prop'， ascending=False)
return group.prop.cumsum().values.searchsorted(q) + 1
diversity = top1000.groupby(['year'， 'sex']).apply(get_quantile_count)
diversity = diversity.unstack('sex')
diversity.head()

diversity.plot(title="Number of popular names in top 50%")

The “Last letter” Revolution
[code]# extract last letter from name column
get_last_letter = lambda x: x[-1]
last_letters = names.name.map(get_last_letter)
last_letters.name = 'last_letter'

table = names.pivot_table('births'， index=last_letters，
columns=['sex'， 'year']， aggfunc=sum)

subtable = table.reindex(columns=[1910， 1960， 2010]， level='year')
subtable.head()

subtable.sum()

letter_prop = subtable / subtable.sum().astype(float)

import matplotlib.pyplot as plt

fig， axes = plt.subplots(2， 1， figsize=(10， 8))
letter_prop['M'].plot(kind='bar'， rot=0， ax=axes[0]， title='Male')
letter_prop['F'].plot(kind='bar'， rot=0， ax=axes[1]， title='Female'，
legend=False)

plt.subplots_adjust(hspace=0.25)

letter_prop = table / table.sum().astype(float)

dny_ts = letter_prop.ix[['d'， 'n'， 'y']， 'M'].T
dny_ts.head()

plt.close('all')

dny_ts.plot()

Boy names that became girl names (and vice versa)
[code]all_names = top1000.name.unique()
mask = np.array(['lesl' in x.lower() for x in all_names])
lesley_like = all_names[mask]
lesley_like

filtered = top1000[top1000.name.isin(lesley_like)]
filtered.groupby('name').births.sum()

table = filtered.pivot_table('births'， index='year'，
columns='sex'， aggfunc='sum')
table = table.div(table.sum(1)， axis=0)
table.tail()

plt.close('all')

table.plot(style={'M': 'k-'， 'F': 'k--'})