使用 Minikube-in-a-Container 和 Jenkins 构建 Istio

docker run --privileged --rm -it quay.io/aspenmesh/minikube-dind
docker exec -it <container> /bin/bash
# kubectl get nodes
<....>
# kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/shell-demo.yaml
# kubectl exec -it shell-demo -- /bin/bash

# Portions Copyright 2016 The Kubernetes Authors All rights reserved.
# Portions Copyright 2018 AspenMesh
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Based on:
# https://github.com/kubernetes/minikube/tree/master/deploy/docker/localkube-dind


FROM debian:jessie

# Install minikube dependencies
RUN DEBIAN_FRONTEND=noninteractive apt-get update -y && \
  DEBIAN_FRONTEND=noninteractive apt-get -yy -q --no-install-recommends install \
  iptables \
  ebtables \
  ethtool \
  ca-certificates \
  conntrack \
  socat \
  git \
  nfs-common \
  glusterfs-client \
  cifs-utils \
  apt-transport-https \
  ca-certificates \
  curl \
  gnupg2 \
  software-properties-common \
  bridge-utils \
  ipcalc \
  aufs-tools \
  sudo \
  && DEBIAN_FRONTEND=noninteractive apt-get clean && \
  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

# Install docker
RUN \
  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
  apt-key export "9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88" | gpg - && \
  echo "deb [arch=amd64] https://download.docker.com/linux/debian jessie stable" >> \
    /etc/apt/sources.list.d/docker.list && \
  DEBIAN_FRONTEND=noninteractive apt-get update && \
  DEBIAN_FRONTEND=noninteractive apt-get -yy -q --no-install-recommends install \
    docker-ce \
  && DEBIAN_FRONTEND=noninteractive apt-get clean && \
  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
VOLUME /var/lib/docker
EXPOSE 2375

# Install minikube
RUN curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.24.1/minikube-linux-amd64 && chmod +x minikube
ENV MINIKUBE_WANTUPDATENOTIFICATION=false
ENV MINIKUBE_WANTREPORTERRORPROMPT=false
ENV CHANGE_MINIKUBE_NONE_USER=true
# minikube --vm-driver=none checks systemctl before starting.  Instead of
# setting up a real systemd environment, install this shim to tell minikube
# what it wants to know: localkube isn't started yet.
COPY fake-systemctl.sh /usr/local/bin/systemctl
EXPOSE 8443

# Install kubectl
RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.9.1/bin/linux/amd64/kubectl && \
  chmod a+x kubectl && \
  mv kubectl /usr/local/bin

# Copy local start.sh
COPY start.sh /start.sh
RUN chmod a+x /start.sh

# If nothing else specified, start up docker and kubernetes.
CMD /start.sh & sleep 4 && tail -F /var/log/docker.log /var/log/dind.log /var/log/minikube-start.log

node('docker') {
  properties([disableConcurrentBuilds()])

  wkdir = "src/istio.io/istio"

  stage('Checkout') {
    checkout scm
  }

  // withRegistry writes to /home/ubuntu/.dockercfg outside of the container
  // (even if you run it inside the docker plugin) which won't be visible
  // inside the builder container, so copy them somewhere that will be
  // visible.  We will symlink to .dockercfg only when needed to reduce
  // the chance of accidentally using the credentials outside of push
  docker.withRegistry('https://quay.io', 'name-of-your-credentials-in-jenkins') {
    stage('Load Push Credentials') {
      sh "cp ~/.dockercfg ${pwd()}/.dockercfg-quay-creds"
    }
  }

  k8sImage = docker.build(
    "k8s-${env.BUILD_TAG}",
    "-f $wkdir/.jenkins/Dockerfile.minikube " +
    "$wkdir/.jenkins/"
  )
  k8sImage.withRun('--privileged') { k8s ->
    stage('Get kubeconfig') {
      sh "docker exec ${k8s.id} /bin/bash -c \"while ! [ -e /kubeconfig ]; do echo waiting for kubeconfig; sleep 3; done\""
      sh "rm -f ${pwd()}/kubeconfig && docker cp ${k8s.id}:/kubeconfig ${pwd()}/kubeconfig"

      // Replace "127.0.0.1" with the path that peer containers can use to
      // get to minikube.
      // minikube will bake certs including the subject "kubernetes" so
      // the kube-api server needs to be reachable from the client's concept
      // of "https://kubernetes:8443" or kubectl will refuse to connect. 
      sh "sed -i'' -e 's;server: https://127.0.0.1:8443;server: https://kubernetes:8443;' kubeconfig"
    }

    builder = docker.build(
      "istio-builder-${env.BUILD_TAG}",
      "-f $wkdir/.jenkins/Dockerfile.jenkins-build " +
        "--build-arg UID=`id -u` --build-arg GID=`id -g` " +
        "$wkdir/.jenkins",
    )

    builder.inside(
      "-e GOPATH=${pwd()} " +
      "-e HOME=${pwd()} " +
      "-e PATH=${pwd()}/bin:\$PATH " +
      "-e KUBECONFIG=${pwd()}/kubeconfig " +
      "-e DOCKER_HOST=\"tcp://kubernetes:2375\" " +
      "--link ${k8s.id}:kubernetes"
    ) {
      stage('Check') {
        sh "ls -al"

        // If there are old credentials from a previous build, destroy them -
        // we will only load them when needed in the push stage
        sh "rm -f ~/.dockercfg"

        sh "cd $wkdir && go get -u github.com/golang/lint/golint"
        sh "cd $wkdir && make check"
      }

      stage('Build') {
        sh "cd $wkdir && make depend"
        sh "cd $wkdir && make build"
      }

      stage('Test') {
        sh "cp kubeconfig $wkdir/pilot/platform/kube/config"
        sh """PROXYVERSION=\$(grep envoy-debug $wkdir/pilot/docker/Dockerfile.proxy_debug  |cut -d: -f2) &&
          PROXY=debug-\$PROXYVERSION &&
          curl -Lo - https://storage.googleapis.com/istio-build/proxy/envoy-\$PROXY.tar.gz | tar xz &&
          mv usr/local/bin/envoy ${pwd()}/bin/envoy &&
          rm -r usr/"""
        sh "cd $wkdir && make test"
      }

      stage('Push') {
        sh "cd && ln -sf .dockercfg-quay-creds .dockercfg"
        sh "cd $wkdir && " +
          "make HUB=yourhub TAG=$BUILD_TAG push"
        gitTag = getTag(wkdir)
        if (gitTag) {
          sh "cd $wkdir && " +
            "make HUB=yourhub TAG=$gitTag push"
        }
        sh "cd && rm .dockercfg"
      }
    }
  }
}

String getTag(String wkdir) {
  return sh(
    script: "cd $wkdir && " +
      "git describe --exact-match --tags \$GIT_COMMIT || true",
    returnStdout: true
  ).trim()
}