Spark: Update to current example standards, add GCS connector

examples/examples_test.go

@@ -344,10 +344,11 @@ func TestExampleObjectSchemas(t *testing.T) {
 			"secret":     &api.Secret{},
 		},
 		"../examples/spark": {
+			"spark-driver-controller": &api.ReplicationController{},
+			"spark-master-controller": &api.ReplicationController{},
 			"spark-master-service":    &api.Service{},
-			"spark-master":            &api.Pod{},
+			"spark-webui":             &api.Service{},
 			"spark-worker-controller": &api.ReplicationController{},
-			"spark-driver":            &api.Pod{},
 		},
 		"../examples/spark/spark-gluster": {
 			"spark-master-service":    &api.Service{},

examples/spark/images/Makefile

 all: push
 
 # To bump the Spark version, bump the version in base/Dockerfile, bump
-# this tag, and bump the uses of this tag in
-# worker/master/driver.
+# this tag and reset to v1. You should also double check the native
+# Hadoop libs at that point (we grab the 2.6.1 libs, which are
+# appropriate for 1.5.1-with-2.6).
 TAG = 1.5.1_v1
 
 containers:

examples/spark/images/base/Dockerfile

@@ -3,17 +3,35 @@ FROM java:latest
 RUN apt-get update -y
 RUN apt-get install -y scala
 
-ENV version 1.5.1
+ENV hadoop_ver 2.6.1
+ENV spark_ver 1.5.1
 
-# Get Spark from some apache mirror.
+# Get Hadoop from US Apache mirror and extract just the native
+# libs. (Until we care about running HDFS with these containers, this
+# is all we need.)
 RUN mkdir -p /opt && \
     cd /opt && \
-    wget http://apache.mirrors.pair.com/spark/spark-${version}/spark-${version}-bin-hadoop2.6.tgz && \
-    tar -zvxf spark-${version}-bin-hadoop2.6.tgz && \
-    rm spark-${version}-bin-hadoop2.6.tgz && \
-    ln -s spark-${version}-bin-hadoop2.6 spark && \
-    echo Spark ${version} installed in /opt
+    wget http://www.us.apache.org/dist/hadoop/common/hadoop-${hadoop_ver}/hadoop-${hadoop_ver}.tar.gz && \
+    tar -zvxf hadoop-${hadoop_ver}.tar.gz hadoop-${hadoop_ver}/lib/native && \
+    rm hadoop-${hadoop_ver}.tar.gz && \
+    ln -s hadoop-${hadoop_ver} hadoop && \
+    echo Hadoop ${hadoop_ver} native libraries installed in /opt/hadoop/lib/native
+
+# Get Spark from US Apache mirror.
+RUN mkdir -p /opt && \
+    cd /opt && \
+    wget http://www.us.apache.org/dist/spark/spark-${spark_ver}/spark-${spark_ver}-bin-hadoop2.6.tgz && \
+    tar -zvxf spark-${spark_ver}-bin-hadoop2.6.tgz && \
+    rm spark-${spark_ver}-bin-hadoop2.6.tgz && \
+    ln -s spark-${spark_ver}-bin-hadoop2.6 spark && \
+    echo Spark ${spark_ver} installed in /opt
+
+# Add the GCS connector.
+RUN wget -O /opt/spark/lib/gcs-connector-latest-hadoop2.jar https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-latest-hadoop2.jar
 
 ADD log4j.properties /opt/spark/conf/log4j.properties
 ADD setup_client.sh /
+ADD start-common.sh /
+ADD core-site.xml /opt/spark/conf/core-site.xml
+ADD spark-defaults.conf /opt/spark/conf/spark-defaults.conf
 ENV PATH $PATH:/opt/spark/bin

examples/spark/images/base/core-site.xml

+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+  <property>
+    <name>fs.gs.impl</name>
+    <value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem</value>
+    <description>The FileSystem for gs: (GCS) uris.</description>
+  </property>
+  <property>
+    <name>fs.AbstractFileSystem.gs.impl</name>
+    <value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</value>
+    <description>The AbstractFileSystem for gs: (GCS) uris. Only necessary for use with Hadoop 2.</description>
+  </property>
+  <property>
+    <name>fs.gs.project.id</name>
+    <value>NOT_RUNNING_INSIDE_GCE</value>
+  </property>
+</configuration>

examples/spark/images/base/spark-defaults.conf

+spark.master spark://spark-master:7077
+spark.executor.extraClassPath /opt/spark/lib/gcs-connector-latest-hadoop2.jar
+spark.driver.extraClassPath   /opt/spark/lib/gcs-connector-latest-hadoop2.jar
+spark.driver.extraLibraryPath /opt/hadoop/lib/native
+spark.app.id KubernetesSpark

examples/spark/images/base/start-common.sh

+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PROJECT_ID=$(curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/project-id)
+
+if [[ -n "${PROJECT_ID}" ]]; then
+  sed -i "s/NOT_RUNNING_INSIDE_GCE/${PROJECT_ID}/" /opt/spark/conf/core-site.xml
+fi
+
+# We don't want any of the incoming service variables, we'd rather use
+# DNS. But this one interferes directly with Spark.
+unset SPARK_MASTER_PORT
+
+# spark.{executor,driver}.extraLibraryPath don't actually seem to
+# work, this seems to be the only reliable way to get the native libs
+# picked up.
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/hadoop/lib/native

examples/spark/images/driver/Dockerfile

-FROM gcr.io/google_containers/spark-base:1.5.1_v1
+FROM gcr.io/google_containers/spark-base:latest
 ADD start.sh /start.sh
 CMD ["/start.sh"]

examples/spark/images/driver/start.sh

@@ -14,9 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-echo "$SPARK_MASTER_SERVICE_HOST spark-master" >> /etc/hosts
-echo "SPARK_LOCAL_HOSTNAME=$(hostname -i)" >> /opt/spark/conf/spark-env.sh
-echo "MASTER=spark://spark-master:$SPARK_MASTER_SERVICE_PORT" >> /opt/spark/conf/spark-env.sh
+. /start-common.sh
+
 echo "Use kubectl exec spark-driver -it bash to invoke commands"
 while true; do
   sleep 100

examples/spark/images/master/Dockerfile

-FROM gcr.io/google_containers/spark-base:1.5.1_v1
+FROM gcr.io/google_containers/spark-base:latest
 
 ADD start.sh /
 ADD log4j.properties /opt/spark/conf/log4j.properties
-EXPOSE 7077
+EXPOSE 7077 8080
 
 ENTRYPOINT ["/start.sh"]

examples/spark/images/master/start.sh

@@ -14,6 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export SPARK_MASTER_PORT=${SPARK_MASTER_SERVICE_PORT:-7077}
+. /start-common.sh
+
+echo "$(hostname -i) spark-master" >> /etc/hosts
+export SPARK_LOCAL_HOSTNAME=spark-master
+export SPARK_MASTER_IP=spark-master
+
 /opt/spark/sbin/start-master.sh
 tail -F /opt/spark/logs/*

examples/spark/images/worker/Dockerfile

-FROM gcr.io/google_containers/spark-base:1.5.1_v1
+FROM gcr.io/google_containers/spark-base:latest
 
 ADD start.sh /
 ADD log4j.properties /opt/spark/conf/log4j.properties

examples/spark/images/worker/start.sh

@@ -14,15 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-if [[ ${SPARK_MASTER_SERVICE_HOST} == "" ]]; then
-  echo "Spark Master service must be created before starting any workers"
-  sleep 30 # To postpone pod restart
-  exit 1
-fi
+. /start-common.sh
 
-echo "${SPARK_MASTER_SERVICE_HOST} spark-master" >> /etc/hosts
-export SPARK_LOCAL_HOSTNAME=$(hostname -i)
-
-/opt/spark/sbin/start-slave.sh spark://spark-master:${SPARK_MASTER_SERVICE_PORT}
+/opt/spark/sbin/start-slave.sh spark://spark-master:7077
 
 tail -F /opt/spark/logs/*

examples/spark/spark-driver-controller.yaml

+kind: ReplicationController
+apiVersion: v1
+metadata:
+  name: spark-driver-controller
+  labels:
+    component: spark-driver
+spec:
+  replicas: 1
+  selector:
+    component: spark-driver
+  template:
+    metadata:
+      labels:
+        component: spark-driver
+    spec:
+      containers:
+        - name: spark-driver
+          image: gcr.io/google_containers/spark-driver:1.5.1_v1
+          resources:
+            requests:
+              cpu: 100m

examples/spark/spark-driver.json

-{
-  "kind": "Pod",
-  "apiVersion": "v1",
-  "metadata": {
-    "name": "spark-driver",
-    "labels": {
-      "name": "spark-driver"
-    }
-  },
-  "spec": {
-    "containers": [
-      {
-        "name": "spark-driver",
-        "image": "gcr.io/google_containers/spark-driver:1.5.1_v1",
-        "resources": {
-          "limits": {
-            "cpu": "100m"
-          }
-        }
-      }
-    ]
-  }
-}

examples/spark/spark-gluster/spark-master-controller.yaml

@@ -3,7 +3,7 @@ apiVersion: v1
 metadata:
   name: spark-master-controller
   labels:
-    component: spark-master-controller
+    component: spark-master
 spec:
   replicas: 1
   selector:
@@ -22,7 +22,7 @@ spec:
             - mountPath: /mnt/glusterfs
               name: glusterfsvol
           resources:
-            limits:
+            requests:
               cpu: 100m
       volumes:
         - name: glusterfsvol

examples/spark/spark-gluster/spark-master-service.yaml

@@ -9,4 +9,4 @@ spec:
     - port: 7077
       targetPort: 7077
   selector:
-    component: spark-master-controller
+    component: spark-master

examples/spark/spark-gluster/spark-worker-controller.yaml

@@ -12,7 +12,7 @@ spec:
     metadata:
       labels:
         component: spark-worker
-        uses: spark-master-controller
+        uses: spark-master
     spec:
       containers:
         - name: spark-worker
@@ -23,7 +23,7 @@ spec:
             - mountPath: /mnt/glusterfs
               name: glusterfsvol
           resources:
-            limits:
+            requests:
               cpu: 100m
       volumes:
         - name: glusterfsvol

examples/spark/spark-master-controller.yaml

+kind: ReplicationController
+apiVersion: v1
+metadata:
+  name: spark-master-controller
+  labels:
+    component: spark-master
+spec:
+  replicas: 1
+  selector:
+    component: spark-master
+  template:
+    metadata:
+      labels:
+        component: spark-master
+    spec:
+      containers:
+        - name: spark-master
+          image: gcr.io/google_containers/spark-master:1.5.1_v1
+          ports:
+            - containerPort: 7077
+            - containerPort: 8080
+          resources:
+            requests:
+              cpu: 100m

examples/spark/spark-master-service.json

-{
-  "kind": "Service",
-  "apiVersion": "v1",
-  "metadata": {
-    "name": "spark-master",
-    "labels": {
-      "name": "spark-master"
-    }
-  },
-  "spec": {
-    "ports": [
-      {
-        "port": 7077,
-        "targetPort": 7077
-      }
-    ],
-    "selector": {
-      "name": "spark-master"
-    }
-  }
-}

examples/spark/spark-master-service.yaml

+kind: Service
+apiVersion: v1
+metadata:
+  name: spark-master
+  labels:
+    component: spark-master-service
+spec:
+  ports:
+    - port: 7077
+      targetPort: 7077
+  selector:
+    component: spark-master

examples/spark/spark-master.json

-{
-  "kind": "Pod",
-  "apiVersion": "v1",
-  "metadata": {
-    "name": "spark-master",
-    "labels": {
-      "name": "spark-master"
-    }
-  },
-  "spec": {
-    "containers": [
-      {
-        "name": "spark-master",
-        "image": "gcr.io/google_containers/spark-master:1.5.1_v1",
-        "ports": [
-          {
-            "containerPort": 7077
-          }
-        ],
-        "resources": {
-          "limits": {
-            "cpu": "100m"
-          }
-        }
-      }
-    ]
-  }
-}

examples/spark/spark-webui.yaml

+kind: Service
+apiVersion: v1
+metadata:
+  name: spark-webui
+spec:
+  ports:
+    - port: 8080
+      targetPort: 8080
+  selector:
+    component: spark-master
+  type: LoadBalancer

examples/spark/spark-worker-controller.json

-{
-  "kind": "ReplicationController",
-  "apiVersion": "v1",
-  "metadata": {
-    "name": "spark-worker-controller",
-    "labels": {
-      "name": "spark-worker"
-    }
-  },
-  "spec": {
-    "replicas": 3,
-    "selector": {
-      "name": "spark-worker"
-    },
-    "template": {
-      "metadata": {
-        "labels": {
-          "name": "spark-worker",
-          "uses": "spark-master"
-        }
-      },
-      "spec": {
-        "containers": [
-          {
-            "name": "spark-worker",
-            "image": "gcr.io/google_containers/spark-worker:1.5.1_v1",
-            "ports": [
-              {
-                "hostPort": 8888,
-                "containerPort": 8888
-              }
-            ],
-            "resources": {
-              "limits": {
-                "cpu": "100m"
-              }
-            }
-          }
-        ]
-      }
-    }
-  }
-}

examples/spark/spark-worker-controller.yaml

+kind: ReplicationController
+apiVersion: v1
+metadata:
+  name: spark-worker-controller
+  labels:
+    component: spark-worker
+spec:
+  replicas: 3
+  selector:
+    component: spark-worker
+  template:
+    metadata:
+      labels:
+        component: spark-worker
+        uses: spark-master
+    spec:
+      containers:
+        - name: spark-worker
+          image: gcr.io/google_containers/spark-worker:1.5.1_v1
+          ports:
+            - containerPort: 8888
+          resources:
+            requests:
+              cpu: 100m