2

3

4

5

6

7

8

9

10

11

12

13

14

15

$ docker pull sequenceiq/spark:1.1.0 $ docker run -i -t -h sandbox sequenceiq/spark \ /etc/bootstrap.sh -bash

$ curl -O http://d3kbcqa49mib13.cloudfront.net/spark-1.1.0.tgz

$ export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=1024M \

-XX:ReservedCodeCacheSize=1024m”

$ mvn -DskipTests clean package

16

sc.parallelize(range(1000)).count()

17

18

19

20

21

ipython notebook --matplotlib inline

22

23

$ ipython profile create pyspark $ vim ~/.ipython/profile_pyspark/ipython_notebook_config.py

24

$ vim ~/.ipython/profile_pyspark/startup/00-pyspark-setup.py

25

import os import sys

os.environ[‘SPARK_HOME’] = ‘/usr/local/spark-1.1.0/‘ sys.path.insert(0, ‘/usr/local/spark-1.1.0/python’) CLUSTER_URL = ‘local’

$ ipython notebook —profile=pyspark OR $ IPYTHON_OPTS="notebook" ./pyspark

26

# dict.txt から Bill が含まれる行を抽出する

words = sc.textFile('/tmp/dict.txt') for word in words.filter(lambda w: 'Bill' in w).take(7): print(word)

27

# dict.txt Bill Nighy Keira Knightley Bill Cosby Rachel McAdams

28

# dict.txt から Bill が含まれる行を抽出する

words = sc.textFile('/tmp/dict.txt') for word in words.filter(lambda w: 'Bill' in w).take(7): print(word)

29

30

31

from pyspark.mllib.classification import NaiveBayesMode

# 流れというか使い方の雰囲気はこんな感じ

model = NaiveBayes.train(sc.parallelize(train_data)) model.predict(features)

32

33

34

35