Rearranged and commented files for running under Jenkins.
This commit is contained in:
Josh Haberman
parent
7d793c167f
commit
67c727cd41
11 changed files with 115 additions and 140 deletions
|
|
@ -10,7 +10,7 @@ os:
|
|||
# The Objective C build needs Xcode 7.0 or later.
|
||||
osx_image: xcode7.2
|
||||
script:
|
||||
- ./tools/run_tests/tests.sh $CONFIG
|
||||
- ./tests.sh $CONFIG
|
||||
env:
|
||||
- CONFIG=cpp
|
||||
- CONFIG=cpp_distcheck
|
||||
|
|
|
|||
6
jenkins/README.md
Normal file
6
jenkins/README.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
Jenkins Infrastructure
|
||||
----------------------
|
||||
|
||||
The scripts in this directory serve as plumbing for running the protobuf
|
||||
tests under Jenkins.
|
||||
56
jenkins/build_and_run_docker.sh
Executable file
56
jenkins/build_and_run_docker.sh
Executable file
|
|
@ -0,0 +1,56 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Builds docker image and runs a command under it.
|
||||
# This is a generic script that is configured with the following variables:
|
||||
#
|
||||
# DOCKERFILE_DIR - Directory in which Dockerfile file is located.
|
||||
# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root)
|
||||
# OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
|
||||
# $@ - Extra args to pass to docker run
|
||||
|
||||
|
||||
set -ex
|
||||
|
||||
cd $(dirname $0)/..
|
||||
git_root=$(pwd)
|
||||
cd -
|
||||
|
||||
# Use image name based on Dockerfile location checksum
|
||||
DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
|
||||
|
||||
# Make sure docker image has been built. Should be instantaneous if so.
|
||||
docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR
|
||||
|
||||
# Ensure existence of ccache directory
|
||||
CCACHE_DIR=/tmp/protobuf-ccache
|
||||
mkdir -p $CCACHE_DIR
|
||||
|
||||
# Choose random name for docker container
|
||||
CONTAINER_NAME="build_and_run_docker_$(uuidgen)"
|
||||
|
||||
# Run command inside docker
|
||||
docker run \
|
||||
"$@" \
|
||||
-e CCACHE_DIR=$CCACHE_DIR \
|
||||
-e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \
|
||||
-e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
|
||||
-v "$git_root:/var/local/jenkins/protobuf:ro" \
|
||||
-v $CCACHE_DIR:$CCACHE_DIR \
|
||||
-w /var/local/git/protobuf \
|
||||
--name=$CONTAINER_NAME \
|
||||
$DOCKER_IMAGE_NAME \
|
||||
bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
|
||||
|
||||
# Copy output artifacts
|
||||
if [ "$OUTPUT_DIR" != "" ]
|
||||
then
|
||||
docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true"
|
||||
fi
|
||||
|
||||
# remove the container, possibly killing it first
|
||||
docker rm -f $CONTAINER_NAME || true
|
||||
|
||||
if [ "$FAILED" != "" ]
|
||||
then
|
||||
exit 1
|
||||
fi
|
||||
6
jenkins/buildcmds/README.md
Normal file
6
jenkins/buildcmds/README.md
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
|
||||
Jenkins Build Commands
|
||||
----------------------
|
||||
|
||||
The scripts in this directory are designed to be top-level entry points for
|
||||
Jenkins projects.
|
||||
15
jenkins/buildcmds/pull_request.sh
Executable file
15
jenkins/buildcmds/pull_request.sh
Executable file
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# This is the top-level script we give to Jenkins as the entry point for
|
||||
# running the "pull request" project:
|
||||
#
|
||||
# https://grpc-testing.appspot.com/view/Protocol%20Buffers/job/protobuf_pull_request/
|
||||
#
|
||||
# This script selects a specific Dockerfile (for building a Docker image) and
|
||||
# a script to run inside that image. Then we delegate to the general
|
||||
# build_and_run_docker.sh script.
|
||||
|
||||
export DOCKERFILE_DIR=jenkins/docker
|
||||
export DOCKER_RUN_SCRIPT=jenkins/pull_request_in_docker.sh
|
||||
export OUTPUT_DIR=testoutput
|
||||
./jenkins/build_and_run_docker.sh
|
||||
|
|
@ -1,31 +1,10 @@
|
|||
# Copyright 2015, Google Inc.
|
||||
# All rights reserved.
|
||||
# This Dockerfile specifies the recipe for creating an image for the tests
|
||||
# to run in.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met:
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above
|
||||
# copyright notice, this list of conditions and the following disclaimer
|
||||
# in the documentation and/or other materials provided with the
|
||||
# distribution.
|
||||
# * Neither the name of Google Inc. nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
# We install as many test dependencies here as we can, because these setup
|
||||
# steps can be cached. They do *not* run every time we run the build.
|
||||
# The Docker image is only rebuilt when the Dockerfile (ie. this file)
|
||||
# changes.
|
||||
|
||||
# Base Dockerfile for gRPC dev images
|
||||
FROM debian:latest
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
"""Gather output from test runs and create an XML file in JUnit format.
|
||||
"""Gathers output from test runs and create an XML file in JUnit format.
|
||||
|
||||
The output files from the individual tests have been written in a directory
|
||||
structure like:
|
||||
|
||||
$DIR/joblog (--joblog from "parallel")
|
||||
$DIR/joblog (output from "parallel --joblog joblog")
|
||||
$DIR/logs/1/cpp/stdout
|
||||
$DIR/logs/1/cpp/stderr
|
||||
$DIR/logs/1/csharp/stdout
|
||||
|
|
@ -13,7 +13,8 @@ structure like:
|
|||
etc.
|
||||
|
||||
This script bundles them into a single output XML file so Jenkins can show
|
||||
detailed test results.
|
||||
detailed test results. It runs as the last step before the Jenkins build
|
||||
finishes.
|
||||
"""
|
||||
|
||||
import os;
|
||||
|
|
@ -1,22 +1,25 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# This is the script that runs inside Docker, once the image has been built,
|
||||
# to execute all tests for the "pull request" project.
|
||||
|
||||
WORKSPACE_BASE=`pwd`
|
||||
MY_DIR="$(dirname "$0")"
|
||||
TEST_SCRIPT=$MY_DIR/tests.sh
|
||||
TEST_SCRIPT=$MY_DIR/../tests.sh
|
||||
BUILD_DIR=/tmp/protobuf
|
||||
|
||||
# Set value used in tests.sh.
|
||||
PARALLELISM=-j8
|
||||
|
||||
set -e # exit immediately on error
|
||||
set -x # display all commands
|
||||
|
||||
# The protobuf repository is mounted into our Docker image, but read-only.
|
||||
# We clone into a directory inside Docker (this is faster than cp).
|
||||
rm -rf $BUILD_DIR
|
||||
mkdir -p $BUILD_DIR
|
||||
cd $BUILD_DIR
|
||||
git clone /var/local/jenkins/protobuf
|
||||
cd protobuf
|
||||
|
||||
# Set up the directory where our test output is going to go.
|
||||
OUTPUT_DIR=`mktemp -d`
|
||||
LOG_OUTPUT_DIR=$OUTPUT_DIR/logs
|
||||
mkdir -p $LOG_OUTPUT_DIR/1/cpp
|
||||
|
|
@ -35,14 +38,14 @@ mkdir -p $LOG_OUTPUT_DIR/1/cpp
|
|||
CPP_STDOUT=$LOG_OUTPUT_DIR/1/cpp/stdout
|
||||
CPP_STDERR=$LOG_OUTPUT_DIR/1/cpp/stderr
|
||||
|
||||
# Time the C++ build, so we can put this info in the test output.
|
||||
# It's important that we get /usr/bin/time (which supports -f and -o) and not
|
||||
# the bash builtin "time" which doesn't.
|
||||
TIME_CMD="/usr/bin/time -f %e -o $LOG_OUTPUT_DIR/1/cpp/build_time"
|
||||
|
||||
$TIME_CMD $TEST_SCRIPT cpp > >(tee $CPP_STDOUT) 2> >(tee $CPP_STDERR >&2)
|
||||
|
||||
# Other tests are run in parallel. The overall run fails if any one of them
|
||||
# fails.
|
||||
# Other tests are run in parallel.
|
||||
|
||||
parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT ::: \
|
||||
csharp \
|
||||
|
|
@ -55,14 +58,15 @@ parallel --results $LOG_OUTPUT_DIR --joblog $OUTPUT_DIR/joblog $TEST_SCRIPT :::
|
|||
ruby21 \
|
||||
|| true # Process test results even if tests fail.
|
||||
|
||||
cat $OUTPUT_DIR/joblog
|
||||
|
||||
# The directory that is copied from Docker back into the Jenkins workspace.
|
||||
COPY_FROM_DOCKER=/var/local/git/protobuf/testoutput
|
||||
mkdir -p $COPY_FROM_DOCKER
|
||||
TESTOUTPUT_XML_FILE=$COPY_FROM_DOCKER/testresults.xml
|
||||
|
||||
python $MY_DIR/../jenkins/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
|
||||
# Process all the output files from "parallel" and package them into a single
|
||||
# .xml file with detailed, broken-down test output.
|
||||
python $MY_DIR/make_test_output.py $OUTPUT_DIR > $TESTOUTPUT_XML_FILE
|
||||
|
||||
ls -l $TESTOUTPUT_XML_FILE
|
||||
|
||||
### disabled tests
|
||||
# java_jdk6 \
|
||||
|
|
@ -1,4 +1,8 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Build and runs tests for the protobuf project. The tests as written here are
|
||||
# used by both Jenkins and Travis, though some specialized logic is required to
|
||||
# handle the differences between them.
|
||||
|
||||
on_travis() {
|
||||
if [ "$TRAVIS" == "true" ]; then
|
||||
|
|
@ -25,19 +29,19 @@ internal_build_cpp() {
|
|||
|
||||
./autogen.sh
|
||||
./configure
|
||||
make $PARALLELISM
|
||||
make -j2
|
||||
}
|
||||
|
||||
build_cpp() {
|
||||
internal_build_cpp
|
||||
make check $PARALLELISM
|
||||
make check -j2
|
||||
cd conformance && make test_cpp && cd ..
|
||||
}
|
||||
|
||||
build_cpp_distcheck() {
|
||||
./autogen.sh
|
||||
./configure
|
||||
make distcheck $PARALLELISM
|
||||
make distcheck -j2
|
||||
}
|
||||
|
||||
build_csharp() {
|
||||
|
|
@ -308,8 +312,6 @@ build_javascript() {
|
|||
cd js && npm install && npm test && cd ..
|
||||
}
|
||||
|
||||
[ -n "${PARALLELISM}" ] && PARALLELISM=-j8
|
||||
|
||||
# Note: travis currently does not support testing more than one language so the
|
||||
# .travis.yml cheats and claims to only be cpp. If they add multiple language
|
||||
# support, this should probably get updated to install steps and/or
|
||||
|
|
@ -320,9 +322,6 @@ build_javascript() {
|
|||
|
||||
# -------- main --------
|
||||
|
||||
# Set value used in tests.sh.
|
||||
PARALLELISM=-j2
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo "
|
||||
Usage: $0 { cpp |
|
||||
|
|
@ -1,84 +0,0 @@
|
|||
#!/bin/bash
|
||||
# Copyright 2016, Google Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are
|
||||
# met:
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above
|
||||
# copyright notice, this list of conditions and the following disclaimer
|
||||
# in the documentation and/or other materials provided with the
|
||||
# distribution.
|
||||
# * Neither the name of Google Inc. nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#
|
||||
# Builds docker image and runs a command under it.
|
||||
# You should never need to call this script on your own.
|
||||
|
||||
set -ex
|
||||
|
||||
cd $(dirname $0)/../..
|
||||
git_root=$(pwd)
|
||||
cd -
|
||||
|
||||
# Inputs
|
||||
# DOCKERFILE_DIR - Directory in which Dockerfile file is located.
|
||||
# DOCKER_RUN_SCRIPT - Script to run under docker (relative to protobuf repo root)
|
||||
# OUTPUT_DIR - Directory that will be copied from inside docker after finishing.
|
||||
# $@ - Extra args to pass to docker run
|
||||
|
||||
# Use image name based on Dockerfile location checksum
|
||||
DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ )
|
||||
|
||||
# Make sure docker image has been built. Should be instantaneous if so.
|
||||
docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR
|
||||
|
||||
# Ensure existence of ccache directory
|
||||
CCACHE_DIR=/tmp/protobuf-ccache
|
||||
mkdir -p $CCACHE_DIR
|
||||
|
||||
# Choose random name for docker container
|
||||
CONTAINER_NAME="build_and_run_docker_$(uuidgen)"
|
||||
|
||||
# Run command inside docker
|
||||
docker run \
|
||||
"$@" \
|
||||
-e CCACHE_DIR=$CCACHE_DIR \
|
||||
-e EXTERNAL_GIT_ROOT="/var/local/jenkins/protobuf" \
|
||||
-e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
|
||||
-v "$git_root:/var/local/jenkins/protobuf:ro" \
|
||||
-v $CCACHE_DIR:$CCACHE_DIR \
|
||||
-w /var/local/git/protobuf \
|
||||
--name=$CONTAINER_NAME \
|
||||
$DOCKER_IMAGE_NAME \
|
||||
bash -l "/var/local/jenkins/protobuf/$DOCKER_RUN_SCRIPT" || FAILED="true"
|
||||
|
||||
# Copy output artifacts
|
||||
if [ "$OUTPUT_DIR" != "" ]
|
||||
then
|
||||
docker cp "$CONTAINER_NAME:/var/local/git/protobuf/$OUTPUT_DIR" "$git_root" || FAILED="true"
|
||||
fi
|
||||
|
||||
# remove the container, possibly killing it first
|
||||
docker rm -f $CONTAINER_NAME || true
|
||||
|
||||
if [ "$FAILED" != "" ]
|
||||
then
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
|
||||
export DOCKERFILE_DIR=tools/docker
|
||||
export DOCKER_RUN_SCRIPT=tools/run_tests/jenkins.sh
|
||||
export OUTPUT_DIR=testoutput
|
||||
./tools/jenkins/build_and_run_docker.sh
|
||||
Loading…
Add table
Reference in a new issue