Skip to content

K
k3s
Commit a4fc22c8 authored by He Simei's avatar He Simei
Browse files
Options

deprecate etcd on node

parent e6e69e31
Hide whitespace changes
Inline Side-by-side
Showing with 90 additions and 210 deletions
cluster/ubuntu/build.sh
View file @ a4fc22c8
......@@ -51,7 +51,6 @@ if [ ! -f etcd.tar.gz ] ; then
tar xzf etcd.tar.gz
fi
cp $ETCD/etcd $ETCD/etcdctl binaries/master
cp $ETCD/etcd $ETCD/etcdctl binaries/minion
# k8s
echo "Download kubernetes release ..."
......
cluster/ubuntu/minion/init_conf/etcd.conf deleted 100644 → 0
View file @ e6e69e31
description "Etcd service"
author "@jainvipin"
start on (net-device-up
and local-filesystems
and runlevel [2345])
respawn
pre-start script
# see also https://github.com/jainvipin/kubernetes-ubuntu-start
ETCD=/opt/bin/$UPSTART_JOB
if [ -f /etc/default/$UPSTART_JOB ]; then
. /etc/default/$UPSTART_JOB
fi
if [ -f $ETCD ]; then
exit 0
fi
echo "$ETCD binary not found, exiting"
exit 22
end script
script
# modify these in /etc/default/$UPSTART_JOB (/etc/default/docker)
ETCD=/opt/bin/$UPSTART_JOB
ETCD_OPTS=""
if [ -f /etc/default/$UPSTART_JOB ]; then
. /etc/default/$UPSTART_JOB
fi
exec "$ETCD" $ETCD_OPTS
end script
cluster/ubuntu/minion/init_conf/flanneld.conf
View file @ a4fc22c8
......@@ -3,10 +3,9 @@ author "@chenxingyu"
respawn
# start in conjunction with etcd
start on started etcd
stop on stopping etcd
start on (net-device-up
and local-filesystems
and runlevel [2345])
pre-start script
FLANNEL=/opt/bin/$UPSTART_JOB
if [ -f /etc/default/$UPSTART_JOB ]; then
......
cluster/ubuntu/minion/init_conf/kube-proxy.conf
View file @ a4fc22c8
......@@ -3,9 +3,9 @@ author "@jainvipin"
respawn
# start in conjunction with etcd
start on started etcd
stop on stopping etcd
# start in conjunction with flanneld
start on started flanneld
stop on stopping flanneld
limit nofile 65536 65536
......
cluster/ubuntu/minion/init_conf/kubelet.conf
View file @ a4fc22c8
......@@ -3,9 +3,9 @@ author "@jainvipin"
respawn
# start in conjunction with etcd
start on started etcd
stop on stopping etcd
# start in conjunction with flanneld
start on started flanneld
stop on stopping flanneld
pre-start script
# see also https://github.com/jainvipin/kubernetes-ubuntu-start
......
cluster/ubuntu/minion/init_scripts/etcd deleted 100755 → 0
View file @ e6e69e31
#!/bin/sh
set -e
### BEGIN INIT INFO
# Provides: etcd
# Required-Start: $docker
# Required-Stop:
# Should-Start:
# Should-Stop:
# Default-Start:
# Default-Stop:
# Short-Description: Start distrubted key/value pair service
# Description:
# http://www.github.com/coreos/etcd
### END INIT INFO
export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/opt/bin:
BASE=$(basename $0)
# modify these in /etc/default/$BASE (/etc/default/etcd)
ETCD=/opt/bin/$BASE
# This is the pid file managed by etcd itself
ETCD_PIDFILE=/var/run/$BASE.pid
ETCD_LOGFILE=/var/log/$BASE.log
ETCD_OPTS=""
ETCD_DESC="Etcd"
# Get lsb functions
. /lib/lsb/init-functions
if [ -f /etc/default/$BASE ]; then
. /etc/default/$BASE
fi
# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it)
if false && [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | grep -q upstart; then
log_failure_msg "$ETCD_DESC is managed via upstart, try using service $BASE $1"
exit 1
fi
# Check etcd is present
if [ ! -x $ETCD ]; then
log_failure_msg "$ETCD not present or not executable"
exit 1
fi
fail_unless_root() {
if [ "$(id -u)" != '0' ]; then
log_failure_msg "$ETCD_DESC must be run as root"
exit 1
fi
}
ETCD_START="start-stop-daemon \
--start \
--background \
--quiet \
--exec $ETCD \
--make-pidfile \
--pidfile $ETCD_PIDFILE \
-- $ETCD_OPTS \
>> $ETCD_LOGFILE 2>&1"
ETCD_STOP="start-stop-daemon \
--stop \
--pidfile $ETCD_PIDFILE"
case "$1" in
start)
fail_unless_root
log_begin_msg "Starting $ETCD_DESC: $BASE"
$ETCD_START
log_end_msg $?
;;
stop)
fail_unless_root
log_begin_msg "Stopping $ETCD_DESC: $BASE"
$ETCD_STOP
log_end_msg $?
;;
restart | force-reload)
fail_unless_root
log_begin_msg "Restarting $ETCD_DESC: $BASE"
$ETCD_STOP
$ETCD_START
log_end_msg $?
;;
status)
status_of_proc -p "$ETCD_PIDFILE" "$ETCD" "$ETCD_DESC"
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
exit 1
;;
esac
cluster/ubuntu/minion/init_scripts/kubelet
View file @ a4fc22c8
......@@ -24,7 +24,7 @@ KUBELET=/opt/bin/$BASE
KUBELET_PIDFILE=/var/run/$BASE.pid
KUBELET_LOGFILE=/var/log/$BASE.log
KUBELET_OPTS=""
KUBELET_DESC="Kube-Apiserver"
KUBELET_DESC="Kubelet"
# Get lsb functions
. /lib/lsb/init-functions
......
cluster/ubuntu/reconfDocker.sh
View file @ a4fc22c8
......@@ -21,33 +21,48 @@ if [ "$(id -u)" != "0" ]; then
exit 1
fi
source ~/kube/config-default.sh
attempt=0
while true; do
/opt/bin/etcdctl get /coreos.com/network/config
if [[ "$?" == 0 ]]; then
break
else
# enough timeout??
if (( attempt > 600 )); then
echo "timeout for waiting network config" > ~/kube/err.log
exit 2
function config_etcd {
source ~/kube/config-default.sh
attempt=0
while true; do
/opt/bin/etcdctl get /coreos.com/network/config
if [[ "$?" == 0 ]]; then
break
else
# enough timeout??
if (( attempt > 600 )); then
echo "timeout for waiting network config" > ~/kube/err.log
exit 2
fi
/opt/bin/etcdctl mk /coreos.com/network/config "{\"Network\":\"${FLANNEL_NET}\"}"
attempt=$((attempt+1))
sleep 3
fi
done
}
/opt/bin/etcdctl mk /coreos.com/network/config "{\"Network\":\"${FLANNEL_NET}\"}"
attempt=$((attempt+1))
sleep 3
fi
done
function restart_docker {
#wait some secs for /run/flannel/subnet.env ready
sleep 15
sudo ip link set dev docker0 down
sudo brctl delbr docker0
#wait some secs for /run/flannel/subnet.env ready
sleep 15
sudo ip link set dev docker0 down
sudo brctl delbr docker0
source /run/flannel/subnet.env
source /run/flannel/subnet.env
echo DOCKER_OPTS=\"${DOCKER_OPTS} -H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock \
--bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}\" > /etc/default/docker
sudo service docker restart
}
echo DOCKER_OPTS=\"${DOCKER_OPTS} -H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock \
--bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}\" > /etc/default/docker
sudo service docker restart
if [[ $1 == "i" ]]; then
restart_docker
elif [[ $1 == "ai" ]]; then
config_etcd
restart_docker
elif [[ $1 == "a" ]]; then
config_etcd
fi
cluster/ubuntu/util.sh
View file @ a4fc22c8
......@@ -21,7 +21,6 @@ SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=E
# use an array to record name and ip
declare -A mm
CLUSTER=""
MASTER=""
MASTER_IP=""
MINION_IPS=""
......@@ -44,28 +43,18 @@ function setClusterInfo() {
MINION_IPS=""
ii=0
for i in $nodes
do
name="infra"$ii
for i in $nodes; do
nodeIP=${i#*@}
item="$name=http://$nodeIP:2380"
if [ "$ii" == 0 ]; then
CLUSTER=$item
else
CLUSTER="$CLUSTER,$item"
fi
mm[$nodeIP]=$name
if [ "${roles[${ii}]}" == "ai" ]; then
if [[ "${roles[${ii}]}" == "ai" ]]; then
MASTER_IP=$nodeIP
MASTER=$i
MINION_IPS="$nodeIP"
elif [ "${roles[${ii}]}" == "a" ]; then
elif [[ "${roles[${ii}]}" == "a" ]]; then
MASTER_IP=$nodeIP
MASTER=$i
elif [ "${roles[${ii}]}" == "i" ]; then
if [ -z "${MINION_IPS}" ];then
elif [[ "${roles[${ii}]}" == "i" ]]; then
if [[ -z "${MINION_IPS}" ]];then
MINION_IPS="$nodeIP"
else
MINION_IPS="$MINION_IPS,$nodeIP"
......@@ -191,12 +180,9 @@ function verify-minion(){
function create-etcd-opts(){
cat <<EOF > ~/kube/default/etcd
ETCD_OPTS="-name $1 \
-initial-advertise-peer-urls http://$2:2380 \
-listen-peer-urls http://$2:2380 \
-initial-cluster-token etcd-cluster-1 \
-initial-cluster $3 \
-initial-cluster-state new"
ETCD_OPTS="-name infra
-listen-client-urls http://0.0.0.0:4001 \
-advertise-client-urls http://127.0.0.1:4001"
EOF
}
......@@ -256,7 +242,7 @@ EOF
function create-flanneld-opts(){
cat <<EOF > ~/kube/default/flanneld
FLANNEL_OPTS=""
FLANNEL_OPTS="--etcd-endpoints=http://${1}:4001"
EOF
}
......@@ -324,10 +310,10 @@ function kube-up() {
{
if [ "${roles[${ii}]}" == "a" ]; then
provision-master
elif [ "${roles[${ii}]}" == "i" ]; then
provision-minion $i
elif [ "${roles[${ii}]}" == "ai" ]; then
provision-masterandminion
elif [ "${roles[${ii}]}" == "i" ]; then
provision-minion $i
else
echo "unsupported role for ${i}. please check"
exit 1
......@@ -356,21 +342,22 @@ function provision-master() {
echo "Deploying master on machine ${MASTER_IP}"
echo
ssh $SSH_OPTS $MASTER "mkdir -p ~/kube/default"
scp -r $SSH_OPTS saltbase/salt/generate-cert/make-ca-cert.sh ubuntu/config-default.sh ubuntu/util.sh ubuntu/master/* ubuntu/binaries/master/ "${MASTER}:~/kube"
scp -r $SSH_OPTS saltbase/salt/generate-cert/make-ca-cert.sh ubuntu/reconfDocker.sh ubuntu/config-default.sh ubuntu/util.sh ubuntu/master/* ubuntu/binaries/master/ "${MASTER}:~/kube"
# remote login to MASTER and use sudo to configue k8s master
ssh $SSH_OPTS -t $MASTER "source ~/kube/util.sh; \
setClusterInfo; \
create-etcd-opts "${mm[${MASTER_IP}]}" "${MASTER_IP}" "${CLUSTER}"; \
create-etcd-opts; \
create-kube-apiserver-opts "${SERVICE_CLUSTER_IP_RANGE}" "${ADMISSION_CONTROL}" "${SERVICE_NODE_PORT_RANGE}"; \
create-kube-controller-manager-opts "${MINION_IPS}"; \
create-kube-scheduler-opts; \
create-flanneld-opts; \
create-flanneld-opts "127.0.0.1"; \
sudo -p '[sudo] password to copy files and start master: ' cp ~/kube/default/* /etc/default/ && sudo cp ~/kube/init_conf/* /etc/init/ && sudo cp ~/kube/init_scripts/* /etc/init.d/ ;\
sudo groupadd -f -r kube-cert; \
sudo ~/kube/make-ca-cert.sh ${MASTER_IP} IP:${MASTER_IP},IP:${SERVICE_CLUSTER_IP_RANGE%.*}.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local; \
sudo mkdir -p /opt/bin/ && sudo cp ~/kube/master/* /opt/bin/; \
sudo service etcd start;"
sudo service etcd start; \
sudo FLANNEL_NET=${FLANNEL_NET} -b ~/kube/reconfDocker.sh "a";"
}
function provision-minion() {
......@@ -383,14 +370,13 @@ function provision-minion() {
# remote login to MASTER and use sudo to configue k8s master
ssh $SSH_OPTS -t $1 "source ~/kube/util.sh; \
setClusterInfo; \
create-etcd-opts "${mm[${1#*@}]}" "${1#*@}" "${CLUSTER}"; \
create-kubelet-opts "${1#*@}" "${MASTER_IP}" "${DNS_SERVER_IP}" "${DNS_DOMAIN}";
create-kubelet-opts "${1#*@}" "${MASTER_IP}" "${DNS_SERVER_IP}" "${DNS_DOMAIN}"; \
create-kube-proxy-opts "${MASTER_IP}"; \
create-flanneld-opts; \
create-flanneld-opts "${MASTER_IP}"; \
sudo -p '[sudo] password to copy files and start minion: ' cp ~/kube/default/* /etc/default/ && sudo cp ~/kube/init_conf/* /etc/init/ && sudo cp ~/kube/init_scripts/* /etc/init.d/ \
&& sudo mkdir -p /opt/bin/ && sudo cp ~/kube/minion/* /opt/bin; \
sudo service etcd start; \
sudo FLANNEL_NET=${FLANNEL_NET} -b ~/kube/reconfDocker.sh"
sudo service flanneld start; \
sudo -b ~/kube/reconfDocker.sh "i";"
}
function provision-masterandminion() {
......@@ -398,24 +384,25 @@ function provision-masterandminion() {
echo "Deploying master and minion on machine ${MASTER_IP}"
echo
ssh $SSH_OPTS $MASTER "mkdir -p ~/kube/default"
scp -r $SSH_OPTS saltbase/salt/generate-cert/make-ca-cert.sh ubuntu/config-default.sh ubuntu/util.sh ubuntu/master/* ubuntu/reconfDocker.sh ubuntu/minion/* ubuntu/binaries/master/ ubuntu/binaries/minion "${MASTER}:~/kube"
# scp order matters
scp -r $SSH_OPTS saltbase/salt/generate-cert/make-ca-cert.sh ubuntu/config-default.sh ubuntu/util.sh ubuntu/minion/* ubuntu/master/* ubuntu/reconfDocker.sh ubuntu/binaries/master/ ubuntu/binaries/minion "${MASTER}:~/kube"
# remote login to the node and use sudo to configue k8s
ssh $SSH_OPTS -t $MASTER "source ~/kube/util.sh; \
setClusterInfo; \
create-etcd-opts "${mm[${MASTER_IP}]}" "${MASTER_IP}" "${CLUSTER}"; \
create-etcd-opts; \
create-kube-apiserver-opts "${SERVICE_CLUSTER_IP_RANGE}" "${ADMISSION_CONTROL}" "${SERVICE_NODE_PORT_RANGE}"; \
create-kube-controller-manager-opts "${MINION_IPS}"; \
create-kube-scheduler-opts; \
create-kubelet-opts "${MASTER_IP}" "${MASTER_IP}" "${DNS_SERVER_IP}" "${DNS_DOMAIN}";
create-kube-proxy-opts "${MASTER_IP}";\
create-flanneld-opts; \
create-flanneld-opts "127.0.0.1"; \
sudo -p '[sudo] password to copy files and start node: ' cp ~/kube/default/* /etc/default/ && sudo cp ~/kube/init_conf/* /etc/init/ && sudo cp ~/kube/init_scripts/* /etc/init.d/ ; \
sudo groupadd -f -r kube-cert; \
sudo ~/kube/make-ca-cert.sh ${MASTER_IP} IP:${MASTER_IP},IP:${SERVICE_CLUSTER_IP_RANGE%.*}.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local; \
sudo mkdir -p /opt/bin/ && sudo cp ~/kube/master/* /opt/bin/ && sudo cp ~/kube/minion/* /opt/bin/; \
sudo service etcd start; \
sudo FLANNEL_NET=${FLANNEL_NET} -b ~/kube/reconfDocker.sh"
sudo FLANNEL_NET=${FLANNEL_NET} -b ~/kube/reconfDocker.sh "ai";"
}
# Delete a kubernetes cluster
......@@ -423,15 +410,26 @@ function kube-down {
KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
source "${KUBE_ROOT}/cluster/ubuntu/${KUBE_CONFIG_FILE-"config-default.sh"}"
ii=0
for i in ${nodes}; do
{
echo "Cleaning on node ${i#*@}"
ssh -t $i 'pgrep etcd && sudo -p "[sudo] password for cleaning etcd data: " service etcd stop && sudo rm -rf /infra*'
# Delete the files in order to generate a clean environment, so you can change each node's role at next deployment.
ssh -t $i 'sudo rm -f /opt/bin/kube* /etc/init/kube* /etc/init.d/kube* /etc/default/kube*; sudo rm -rf ~/kube /var/lib/kubelet'
if [[ "${roles[${ii}]}" == "ai" || "${roles[${ii}]}" == "a" ]]; then
ssh -t $i 'pgrep etcd && sudo -p "[sudo] password for cleaning etcd data: " service etcd stop && sudo rm -rf /infra*;
sudo rm -rf /opt/bin/etcd* /etc/init/etcd.conf /etc/init.d/etcd /etc/default/etcd'
elif [[ "${roles[${ii}]}" == "i" ]]; then
ssh -t $i 'pgrep flanneld && sudo -p "[sudo] password for stopping flanneld: " service flanneld stop'
else
echo "unsupported role for ${i}"
fi
# Delete the files in order to generate a clean environment, so you can change each node's role at next deployment.
ssh -t $i 'sudo rm -f /opt/bin/kube* /opt/bin/flanneld;
sudo rm -rf /etc/init/kube* /etc/init/flanneld.conf /etc/init.d/kube* /etc/init.d/flanneld;
sudo rm -rf /etc/default/kube* /etc/default/flanneld;
sudo rm -rf ~/kube /var/lib/kubelet'
}
((ii=ii+1))
done
wait
}
# Update a kubernetes cluster with latest source
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment