Skip to content
Snippets Groups Projects
Commit cdcb1085 authored by Ambrus Simon's avatar Ambrus Simon
Browse files

Enable build/lint/unit/integ skip and py.test args for individual test running

parent 605ffa59
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.pylintrc
+ 1
− 1
View file @ cdcb1085
......@@ -212,7 +212,7 @@ indent-string=' '
indent-after-paren=4
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
expected-line-ending-format=
expected-line-ending-format=LF
[LOGGING]
......
TESTING.md
+ 14
− 4
View file @ cdcb1085
## Run the tests
### OSX
```
./test/bin/run-tests-osx.sh
```
### Ubuntu
Run automated tests:
```
# Follow installation instructions in README first
. /runtime/bin/activate # Or wherever your scitran virtualenv is
./test/bin/setup-integration-tests-ubuntu.sh
./test/bin/run-tests-ubuntu.sh
```
* To skip linting, use `--no-lint` (`-L`)
* To skip unit tests, use `--no-unit` (`-U`)
* To skip integration tests, use `--no-integ` (`-I`)
* To pass any arguments to `py.test`, use `-- PYTEST_ARGS`
### Docker
Build scitran-core image and run automated tests in a docker container:
```
./test/bin/run-tests-docker.sh
```
* To skip building the image, use `--no-build` (`-B`)
* To pass any arguments to `run-tests-ubuntu.sh`, use `-- TEST_ARGS`
#### Example
Without rebuilding the image, run only unit tests matching `foo`, use the highest verbosity level for test output and jump into a python debugger session in case an assertion fails:
```
./test/bin/run-tests-docker.sh -B -- -L -I -- -k foo -vvv --pdb
```
### Tools
- [abao](https://github.com/cybertk/abao/)
......
test/bin/lint.sh deleted 100755 → 0
+ 0
− 21
View file @ 605ffa59
#!/usr/bin/env bash
set -eu
unset CDPATH
cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
echo "Checking for files with DOS encoding:"
(! git ls-files | xargs file | grep -I "with CRLF line terminators")
echo "Checking for files with windows-style newlines:"
(! git ls-files | xargs grep -I $'\r')
echo "Running pylint ..."
# TODO: Enable Refactor and Convention reports
pylint --reports=no --disable=C,R,W0312,W0141,W0110 api
#echo
#
#echo "Running pep8 ..."
#pep8 --max-line-length=150 --ignore=E402 api
test/bin/run-integration-tests.sh deleted 100755 → 0
+ 0
− 78
View file @ 605ffa59
#!/usr/bin/env bash
set -eu
unset CDPATH
cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
rm -f .coverage.integration-tests
USAGE="
Usage:\n
$0 <api-base-url> <mongodb-uri> <mongodb-log-uri> <drone-secret>\n
\n
"
if [ "$#" -eq 4 ]; then
SCITRAN_SITE_API_URL=$1
SCITRAN_PERSISTENT_DB_URI=$2
SCITRAN_PERSISTENT_DB_LOG_URI=$3
SCITRAN_CORE_DRONE_SECRET=$4
else
echo "Wrong number of positional arguments"
echo $USAGE >&2
exit 1
fi
echo "Connecting to API"
until $(curl --output /dev/null --silent --head --fail "$SCITRAN_SITE_API_URL"); do
printf '.'
sleep 1
done
# Remove __pycache__ directory for issue with __file__ attribute
# Due to running the tests on the host creating bytecode files
# Which have a mismatched __file__ attribute when loaded in docker container
rm -rf test/integration_tests/python/__pycache__
PYTHONPATH="$( pwd )" \
SCITRAN_SITE_API_URL="$SCITRAN_SITE_API_URL" \
SCITRAN_PERSISTENT_DB_URI="$SCITRAN_PERSISTENT_DB_URI" \
SCITRAN_PERSISTENT_DB_LOG_URI="$SCITRAN_PERSISTENT_DB_LOG_URI" \
SCITRAN_CORE_DRONE_SECRET="$SCITRAN_CORE_DRONE_SECRET" \
py.test test/integration_tests/python
# Create resources that Abao relies on:
# - user w/ api key
# - scitran group
# - test-group
# - test-project-1 (+analysis upload)
# - test-session-1 (+analysis upload)
# - test-acquisition-1 (+analysis upload)
# - test-case-gear
# - test-collection-1 (+analysis upload)
SCITRAN_SITE_API_URL="$SCITRAN_SITE_API_URL" \
SCITRAN_CORE_DRONE_SECRET="$SCITRAN_CORE_DRONE_SECRET" \
SCITRAN_PERSISTENT_DB_URI="$SCITRAN_PERSISTENT_DB_URI" \
python test/integration_tests/abao/load_fixture.py
set +u
# If no VIRTUAL_ENV, make sure /usr/local/bin is in the path
if [ -z "$VIRTUAL_ENV" ]; then
PATH="/usr/local/bin:$PATH"
npm install test/integration_tests
else
npm install --global test/integration_tests
fi
set -u
PATH="$(npm bin):$PATH"
# Allow us to require modules from package.json,
# since abao_test_hooks.js is not being called from the package directory
integration_test_node_modules="$( pwd )/node_modules/scitran-core-integration-tests/node_modules"
# Have to change into definitions directory to resolve
# relative $ref's in the jsonschema's
pushd raml/schemas/definitions
NODE_PATH="$integration_test_node_modules" abao ../../api.raml "--server=$SCITRAN_SITE_API_URL" "--hookfiles=../../../test/integration_tests/abao/abao_test_hooks.js"
popd
test/bin/run-tests-docker.sh
+ 74
− 78
View file @ cdcb1085
#!/usr/bin/env bash
set -e
set -eu
unset CDPATH
cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
IMAGE_NAME_SCITRAN_CORE="scitran-core:run-tests"
IMAGE_NAME_MONGO=mongo
CONTAINER_NAME_MONGO=scitran-core-test-mongo
CONTAINER_NAME_SCITRAN_CORE=scitran-core-test-uwsgi
USAGE="
Run scitran-core tests using docker
\n
Usage:\n
\n
--help: print help and exit\n
-b, --build-image: Rebuild scitran-core base image\n
-L, --no-lint: Skip linter\n
"
SCITRAN_RUN_LINT="true"
BUILD_IMAGE="false"
while [ "$#" -gt 0 ]; do
key="$1"
case $key in
--help)
echo -e $USAGE >&2
exit 1
;;
-b|--build-image)
BUILD_IMAGE="true"
;;
-L|--no-lint)
SCITRAN_RUN_LINT="false"
;;
*)
echo "Invalid option: $key" >&2
echo -e $USAGE >&2
exit 1
;;
esac
shift
done
clean_up () {
# Copy coverage file to host for possible further reporting
docker cp "$CONTAINER_NAME_SCITRAN_CORE":/var/scitran/code/api/.coverage .coverage || true
# Stop and remove containers
docker rm -v -f "$CONTAINER_NAME_MONGO"
docker rm -v -f "$CONTAINER_NAME_SCITRAN_CORE"
function usage() {
cat >&2 <<EOF
Build scitran-core image and run tests in a docker container
Usage:
$0 [OPTION...] [-- TEST_ARGS...]
Options:
-B, --no-build Skip docker build
-h, --help Print this help and exit
-- TEST_ARGS Arguments passed to test/bin/run-tests-ubuntu.sh
EOF
}
function main() {
local DOCKER_BUILD=true
local TEST_ARGS=
while [[ "$#" > 0 ]]; do
case "$1" in
-B|--no-build) DOCKER_BUILD=false; ;;
-h|--help) usage; exit 0 ;;
--) TEST_ARGS="${@:2}"; break ;;
*) echo "Invalid argument: $1" >&2; usage; exit 1 ;;
esac
shift
done
if ${DOCKER_BUILD}; then
echo "Building scitran-core:run-tests ..."
docker build -t scitran-core:run-tests .
fi
docker network create scitran-core-test-network
# Launch Mongo instance
docker run -d \
--name scitran-core-test-mongo \
--network scitran-core-test-network \
mongo
# Execute tests
docker run -it \
--name scitran-core-test-uwsgi \
--network scitran-core-test-network \
-e SCITRAN_PERSISTENT_DB_URI=mongodb://scitran-core-test-mongo:27017/scitran \
-e SCITRAN_PERSISTENT_DB_LOG_URI=mongodb://scitran-core-test-mongo:27017/logs \
-v $(pwd):/var/scitran/code/api \
--entrypoint bash \
scitran-core:run-tests \
/var/scitran/code/api/test/bin/run-tests-ubuntu.sh \
$TEST_ARGS
}
function clean_up() {
export TEST_RESULT_CODE=$?
set +e
# Copy coverage file to host for possible further reporting
docker cp scitran-core-test-uwsgi:/var/scitran/code/api/.coverage .coverage
# Spin down dependencies
docker rm -f -v scitran-core-test-uwsgi
docker rm -f -v scitran-core-test-mongo
docker network rm scitran-core-test-network
exit $TEST_RESULT_CODE
}
trap clean_up EXIT
if [[ $( docker images "$IMAGE_NAME_SCITRAN_CORE" | tail -n +2 ) == "" ]]; then
echo "$IMAGE_NAME_SCITRAN_CORE image not found. Building"
BUILD_IMAGE="true"
fi
if [ "$BUILD_IMAGE" == "true" ]; then
docker build -t "$IMAGE_NAME_SCITRAN_CORE" .
fi
# Sub-shell the test steps to make the functionality of the trap execution explicit
(
# Launch Mongo isinstance
docker run --name "$CONTAINER_NAME_MONGO" -d "$IMAGE_NAME_MONGO"
# Execute tests
docker run \
-it \
--name "$CONTAINER_NAME_SCITRAN_CORE"\
-e "SCITRAN_PERSISTENT_DB_URI=mongodb://$CONTAINER_NAME_MONGO:27017/scitran" \
-e "SCITRAN_PERSISTENT_DB_LOG_URI=mongodb://$CONTAINER_NAME_MONGO:27017/logs" \
-e "SCITRAN_RUN_LINT=$SCITRAN_RUN_LINT" \
--link "$CONTAINER_NAME_MONGO" \
-v $(pwd):/var/scitran/code/api \
--entrypoint bash \
"$IMAGE_NAME_SCITRAN_CORE" \
/var/scitran/code/api/test/bin/run-tests-ubuntu.sh
)
main "$@"
test/bin/run-tests-osx.sh deleted 100755 → 0
+ 0
− 65
View file @ 605ffa59
#!/usr/bin/env bash
set -e
unset CDPATH
cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
set -a
# Use port 9003 to hopefully avoid conflicts
SCITRAN_RUNTIME_PATH=${SCITRAN_RUNTIME_PATH:-"$( pwd )/runtime"}
SCITRAN_PERSISTENT_DB_PORT=9003
SCITRAN_PERSISTENT_DB_URI="mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/integration-tests"
SCITRAN_PERSISTENT_DB_LOG_URI=${SCITRAN_PERSISTENT_DB_LOG_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/logs"}
SCITRAN_PERSISTENT_PATH="$( mktemp -d )"
SCITRAN_CORE_DRONE_SECRET=${SCITRAN_CORE_DRONE_SECRET:-$( openssl rand -base64 32 )}
clean_up () {
kill $API_PID || true
wait 2> /dev/null
rm -rf "$SCITRAN_PERSISTENT_PATH"
# NOTE on omit: cross-site feature unused and planned for removal
local OMIT="--omit api/centralclient.py"
echo -e "\nUNIT TEST COVERAGE:"
coverage report $OMIT --skip-covered
coverage combine
echo -e "\nOVERALL COVERAGE:"
coverage report $OMIT --show-missing
coverage html $OMIT
}
trap clean_up EXIT
./bin/install-dev-osx.sh
source $SCITRAN_RUNTIME_PATH/bin/activate # will fail with `set -u`
# Install Node.js
if [ ! -f "$SCITRAN_RUNTIME_PATH/bin/node" ]; then
echo "Installing Node.js"
NODE_URL="https://nodejs.org/dist/v6.10.2/node-v6.10.2-darwin-x64.tar.gz"
curl $NODE_URL | tar xz -C $VIRTUAL_ENV --strip-components 1
fi
# Install testing dependencies
echo "Installing testing dependencies"
pip install --no-cache-dir -r "test/integration_tests/requirements-integration-test.txt"
./test/bin/lint.sh api
SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET \
./test/bin/run-unit-tests.sh
SCITRAN_RUNTIME_PORT=8081 \
SCITRAN_CORE_DRONE_SECRET="$SCITRAN_CORE_DRONE_SECRET" \
SCITRAN_RUNTIME_COVERAGE="true" \
./bin/run-dev-osx.sh -T -U -I &
API_PID=$!
./test/bin/run-integration-tests.sh \
"http://localhost:8081/api" \
"$SCITRAN_PERSISTENT_DB_URI" \
"$SCITRAN_PERSISTENT_DB_LOG_URI" \
"$SCITRAN_CORE_DRONE_SECRET"
test/bin/run-tests-ubuntu.sh
+ 153
− 64
View file @ cdcb1085
#!/usr/bin/env bash
#
# Run all scripted tests from ubuntu 14.04 or later
#
# Assumes mongo db instance is accessible at localhost, unless
# SCITRAN_PERSISTENT_DB_URI or SCITRAN_PERSISTENT_DB_LOG_URI specify otherwise.
#
# Forces SCITRAN_CORE_ACCESS_LOG_ENABLED=true
#
# Variables:
# - See sample.config for options.
# - SCITRAN_PERSISTENT_DB_URI will cause SCITRAN_PERSISTENT_DB_PORT to be
# ignored.
# - SCITRAN_PERSISTENT_DB_LOG_URI will cause SCITRAN_PERSISTENT_DB_PORT to be
# ignored.
set -e
set -eu
unset CDPATH
cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
SCITRAN_RUN_LINT=${SCITRAN_RUN_LINT:-"true"}
if [ "$SCITRAN_RUN_LINT" == "true" ]; then
./test/bin/lint.sh api
fi
function usage() {
cat >&2 <<EOF
Run scitran-core tests
Usage:
$0 [OPTION...]
clean_up () {
kill $API_PID || true
wait 2> /dev/null
Options:
-L, --no-lint Skip linting
-U, --no-unit Skip unit tests
-I, --no-integ Skip integration tests
-h, --help Print this help and exit
-- PYTEST_ARGS Arguments passed to py.test
# NOTE on omit: cross-site feature unused and planned for removal
local OMIT="--omit api/centralclient.py"
echo -e "\nUNIT TEST COVERAGE:"
coverage report $OMIT --skip-covered
Envvars:
SCITRAN_PERSISTENT_DB_PORT (9001)
SCITRAN_PERSISTENT_DB_URI (mongodb://localhost:9001/scitran)
SCITRAN_PERSISTENT_DB_LOG_URI (mongodb://localhost:9001/logs)
coverage combine
echo -e "\nOVERALL COVERAGE:"
coverage report $OMIT --show-missing
coverage html $OMIT
Assumes mongo db instance is accessible at localhost, unless
SCITRAN_PERSISTENT_DB_URI or SCITRAN_PERSISTENT_DB_LOG_URI specify otherwise.
EOF
}
function main() {
local RUN_LINT=true
local RUN_UNIT=true
local RUN_INTEG=true
local PYTEST_ARGS=
while [[ "$#" > 0 ]]; do
case "$1" in
-L|--no-lint) RUN_LINT=false ;;
-U|--no-unit) RUN_UNIT=false ;;
-I|--no-integ) RUN_INTEG=false ;;
-h|--help) usage; exit 0 ;;
--) PYTEST_ARGS="${@:2}"; break ;;
*) echo "Invalid argument: $1" >&2; usage; exit 1 ;;
esac
shift
done
# Remove __pycache__ directories for issue with __file__ attribute due to
# running the tests on the host creating bytecode files hich have a
# mismatched __file__ attribute when loaded in docker container
rm -rf test/unit_tests/python/__pycache__
rm -rf test/integration_tests/python/__pycache__
export PYTHONPATH="$(pwd)"
export SCITRAN_SITE_API_URL="http://localhost:8081/api"
export SCITRAN_PERSISTENT_DB_PORT=${SCITRAN_PERSISTENT_DB_PORT:-"9001"}
export SCITRAN_PERSISTENT_DB_URI=${SCITRAN_PERSISTENT_DB_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"}
export SCITRAN_PERSISTENT_DB_LOG_URI=${SCITRAN_PERSISTENT_DB_LOG_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/logs"}
export SCITRAN_PERSISTENT_PATH=`mktemp -d`
export SCITRAN_PERSISTENT_DATA_PATH="$SCITRAN_PERSISTENT_PATH/data"
export SCITRAN_CORE_DRONE_SECRET=${SCITRAN_CORE_DRONE_SECRET:-$( openssl rand -base64 32 )}
if ${RUN_LINT}; then
echo "Running pylint ..."
# TODO Enable Refactor and Convention reports
# TODO Move --disable into rc
pylint --reports=no --disable=C,R,W0312,W0141,W0110 api
# echo "Running pep8 ..."
# pep8 --max-line-length=150 --ignore=E402 api
fi
if ${RUN_UNIT}; then
echo "Running unit tests ..."
rm -f .coverage
py.test --cov=api --cov-report= test/unit_tests/python $PYTEST_ARGS
fi
if ${RUN_INTEG}; then
echo "Running integration tests ..."
uwsgi --http "localhost:8081" --master --http-keepalive \
--so-keepalive --add-header "Connection: Keep-Alive" \
--processes 1 --threads 1 \
--enable-threads \
--wsgi-file bin/api.wsgi \
--die-on-term \
--logformat '%(addr) - %(user) [%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)" request_id=%(request_id)' \
--env "SCITRAN_PERSISTENT_DB_URI=$SCITRAN_PERSISTENT_DB_URI" \
--env "SCITRAN_PERSISTENT_DB_LOG_URI=$SCITRAN_PERSISTENT_DB_LOG_URI" \
--env "SCITRAN_PERSISTENT_PATH=$SCITRAN_PERSISTENT_PATH" \
--env "SCITRAN_PERSISTENT_DATA_PATH=$SCITRAN_PERSISTENT_DATA_PATH" \
--env "SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET" \
--env "SCITRAN_RUNTIME_COVERAGE=true" \
--env "SCITRAN_CORE_ACCESS_LOG_ENABLED=true" &
export API_PID=$!
echo "Connecting to API"
until $(curl --output /dev/null --silent --head --fail "$SCITRAN_SITE_API_URL"); do
printf '.'
sleep 1
done
py.test test/integration_tests/python $PYTEST_ARGS
# Create resources that Abao relies on
python test/integration_tests/abao/load_fixture.py
# If no VIRTUAL_ENV, make sure /usr/local/bin is in the path
if [[ -z "${VIRTUAL_ENV:-}" ]]; then
PATH="/usr/local/bin:$PATH"
npm install test/integration_tests
else
npm install --global test/integration_tests
fi
PATH="$(npm bin):$PATH"
# Allow us to require modules from package.json,
# since abao_test_hooks.js is not being called from the package directory
integration_test_node_modules="$(pwd)/node_modules/scitran-core-integration-tests/node_modules"
# Have to change into definitions directory to resolve
# relative $ref's in the jsonschema's
pushd raml/schemas/definitions
NODE_PATH="$integration_test_node_modules" abao ../../api.raml "--server=$SCITRAN_SITE_API_URL" "--hookfiles=../../../test/integration_tests/abao/abao_test_hooks.js"
popd
fi
}
function clean_up () {
export TEST_RESULT_CODE=$?
set +e
echo
echo "Test return code = $TEST_RESULT_CODE"
if [[ -n "${API_PID:-}" ]]; then
# Killing uwsgi
kill $API_PID || true
wait 2> /dev/null
fi
if [[ "${TEST_RESULT_CODE}" == "0" ]]; then
echo
echo "UNIT TEST COVERAGE:"
coverage report --skip-covered
echo
echo "OVERALL COVERAGE:"
coverage combine
coverage report --show-missing
coverage html
fi
exit $TEST_RESULT_CODE
}
trap clean_up EXIT
API_BASE_URL="http://localhost:8081/api"
SCITRAN_PERSISTENT_DB_PORT=${SCITRAN_PERSISTENT_DB_PORT:-"9001"}
SCITRAN_PERSISTENT_DB_URI=${SCITRAN_PERSISTENT_DB_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/scitran"}
SCITRAN_PERSISTENT_DB_LOG_URI=${SCITRAN_PERSISTENT_DB_LOG_URI:-"mongodb://localhost:$SCITRAN_PERSISTENT_DB_PORT/logs"}
SCITRAN_PERSISTENT_PATH=`mktemp -d`
SCITRAN_PERSISTENT_DATA_PATH="$SCITRAN_PERSISTENT_PATH/data"
SCITRAN_CORE_DRONE_SECRET=${SCITRAN_CORE_DRONE_SECRET:-$( openssl rand -base64 32 )}
SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET \
./test/bin/run-unit-tests.sh
uwsgi --http "localhost:8081" --master --http-keepalive \
--so-keepalive --add-header "Connection: Keep-Alive" \
--processes 1 --threads 1 \
--enable-threads \
--wsgi-file bin/api.wsgi \
--die-on-term \
--logformat '%(addr) - %(user) [%(ltime)] "%(method) %(uri) %(proto)" %(status) %(size) "%(referer)" "%(uagent)" request_id=%(request_id)' \
--env "SCITRAN_PERSISTENT_DB_URI=$SCITRAN_PERSISTENT_DB_URI" \
--env "SCITRAN_PERSISTENT_DB_LOG_URI=$SCITRAN_PERSISTENT_DB_LOG_URI" \
--env "SCITRAN_PERSISTENT_PATH=$SCITRAN_PERSISTENT_PATH" \
--env "SCITRAN_PERSISTENT_DATA_PATH=$SCITRAN_PERSISTENT_DATA_PATH" \
--env "SCITRAN_CORE_DRONE_SECRET=$SCITRAN_CORE_DRONE_SECRET" \
--env 'SCITRAN_RUNTIME_COVERAGE=true' \
--env 'SCITRAN_CORE_ACCESS_LOG_ENABLED=true' &
API_PID=$!
./test/bin/run-integration-tests.sh \
"$API_BASE_URL" \
"$SCITRAN_PERSISTENT_DB_URI" \
"$SCITRAN_PERSISTENT_DB_LOG_URI" \
"$SCITRAN_CORE_DRONE_SECRET"
main "$@"
test/bin/run-unit-tests.sh deleted 100755 → 0
+ 0
− 13
View file @ 605ffa59
set -e
unset CDPATH
cd "$( dirname "${BASH_SOURCE[0]}" )/../.."
# Remove __pycache__ directory for issue with __file__ attribute
# Due to running the tests on the host creating bytecode files
# Which have a mismatched __file__ attribute when loaded in docker container
rm -rf test/unit_tests/python/__pycache__
rm -f .coverage
PYTHONPATH="$( pwd )" py.test --cov=api --cov-report= test/unit_tests/python
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment