From e9314e5b30ac6ed68a4c768c30baf2047d108eb7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 10 Oct 2017 11:02:39 +0100 Subject: [PATCH] Untangle precommit and travis test scripts (#288) --- .travis.yml | 4 +- INSTALL.md | 4 +- hooks/pre-commit | 31 +------------- scripts/README.md | 7 ++++ scripts/build-test-lint.sh | 26 ++++++++++++ scripts/find-lint.sh | 41 +++++++++++++++++++ .../install-local-kafka.sh | 2 + travis-test.sh => scripts/travis-test.sh | 10 ++++- 8 files changed, 89 insertions(+), 36 deletions(-) create mode 100644 scripts/README.md create mode 100755 scripts/build-test-lint.sh create mode 100755 scripts/find-lint.sh rename travis-install-kafka.sh => scripts/install-local-kafka.sh (95%) rename travis-test.sh => scripts/travis-test.sh (78%) diff --git a/.travis.yml b/.travis.yml index ab2492d2..dd7359af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,8 +24,8 @@ before_script: - openssl req -x509 -newkey rsa:4096 -keyout server.key -out server.crt -days 365 -nodes -subj /CN=localhost script: - - ./travis-install-kafka.sh - - ./travis-test.sh + - ./scripts/install-local-kafka.sh + - ./scripts/travis-test.sh notifications: webhooks: diff --git a/INSTALL.md b/INSTALL.md index 1d002c5f..54addd28 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -17,7 +17,7 @@ Dendrite can be run in one of two configurations: - For Kafka (optional if using the monolith server): - Unix-based system (https://kafka.apache.org/documentation/#os) - JDK 1.8+ / OpenJDK 1.8+ - - Apache Kafka 0.10.2+ (see [travis-install-kafka.sh](travis-install-kafka.sh) for up-to-date version numbers) + - Apache Kafka 0.10.2+ (see [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh) for up-to-date version numbers) ## Setting up a development environment @@ -34,7 +34,7 @@ go get github.com/constabulary/gb/... gb build ``` -If using Kafka, install and start it (c.f. [travis-install-kafka.sh](travis-install-kafka.sh)): +If using Kafka, install and start it (c.f. [scripts/install-local-kafka.sh](scripts/install-local-kafka.sh)): ```bash MIRROR=http://apache.mirror.anlx.net/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz diff --git a/hooks/pre-commit b/hooks/pre-commit index 904d38dc..ec7e3e1e 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -2,33 +2,4 @@ set -eu -# Tune the GC to use more memory to reduce the number of garbage collections -export GOGC=400 -export GOPATH="$(pwd):$(pwd)/vendor" -export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin" - -echo "Checking that it builds" -gb build - -# Check that all the packages can build. -# When `go build` is given multiple packages it won't output anything, and just -# checks that everything builds. This seems to do a better job of handling -# missing imports than `gb build` does. -echo "Double checking it builds..." -go build github.com/matrix-org/dendrite/cmd/... - -echo "Installing lint search engine..." -go install github.com/alecthomas/gometalinter/ -gometalinter --config=linter.json ./... --install - -echo "Looking for lint..." -gometalinter --config=linter.json ./... --enable-gc - -echo "Double checking spelling..." -misspell -error src *.md - -echo "Testing..." -gb test - - -echo "Done!" +./scripts/build-test-lint.sh diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..4d855ba4 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,7 @@ +# Dev Scripts + +These are a collection of scripts that should be helpful for those developing +on dendrite. + +See `find-lint.sh` for environment variables that control linter resource +usage. diff --git a/scripts/build-test-lint.sh b/scripts/build-test-lint.sh new file mode 100755 index 00000000..cf1f37b7 --- /dev/null +++ b/scripts/build-test-lint.sh @@ -0,0 +1,26 @@ +#! /bin/bash + +# Builds, tests and lints dendrite, and should be run before pushing commits + +set -eu + +export GOPATH="$(pwd):$(pwd)/vendor" +export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin" + +echo "Checking that it builds" +gb build + +# Check that all the packages can build. +# When `go build` is given multiple packages it won't output anything, and just +# checks that everything builds. This seems to do a better job of handling +# missing imports than `gb build` does. +echo "Double checking it builds..." +go build github.com/matrix-org/dendrite/cmd/... + +./scripts/find-lint.sh + +echo "Double checking spelling..." +misspell -error src *.md + +echo "Testing..." +gb test diff --git a/scripts/find-lint.sh b/scripts/find-lint.sh new file mode 100755 index 00000000..81ca806b --- /dev/null +++ b/scripts/find-lint.sh @@ -0,0 +1,41 @@ +#! /bin/bash + +# Runs the linters against dendrite + +# The linters can take a lot of resources and are slow, so they can be +# configured using two environment variables: +# +# - `DENDRITE_LINT_CONCURRENCY` - number of concurrent linters to run, +# gometalinter defaults this to 8 +# - `DENDRITE_LINT_DISABLE_GC` - if set then the the go gc will be disabled +# when running the linters, speeding them up but using much more memory. + + +set -eu + +export GOPATH="$(pwd):$(pwd)/vendor" +export PATH="$PATH:$(pwd)/vendor/bin:$(pwd)/bin" + +args="" +if [ ${1:-""} = "fast" ] +then args="--config=linter-fast.json" +else args="--config=linter.json" +fi + +if [ -n "${DENDRITE_LINT_CONCURRENCY:-}" ] +then args="$args --concurrency=$DENDRITE_LINT_CONCURRENCY" +fi + +if [ -z "${DENDRITE_LINT_DISABLE_GC:-}" ] +then args="$args --enable-gc" +fi + +echo "Installing lint search engine..." +go install github.com/alecthomas/gometalinter/ +gometalinter --config=linter.json ./... --install + +echo "Looking for lint..." +gometalinter ./... $args + +echo "Double checking spelling..." +misspell -error src *.md diff --git a/travis-install-kafka.sh b/scripts/install-local-kafka.sh similarity index 95% rename from travis-install-kafka.sh rename to scripts/install-local-kafka.sh index 32f95234..869d2891 100755 --- a/travis-install-kafka.sh +++ b/scripts/install-local-kafka.sh @@ -1,5 +1,7 @@ # /bin/bash +# Downloads, installs and runs a kafka instance + set -eu # The mirror to download kafka from is picked from the list of mirrors at diff --git a/travis-test.sh b/scripts/travis-test.sh similarity index 78% rename from travis-test.sh rename to scripts/travis-test.sh index 20d5f6fa..5851c3b0 100755 --- a/travis-test.sh +++ b/scripts/travis-test.sh @@ -1,7 +1,13 @@ #! /bin/bash +# The entry point for travis tests + set -eu +# Tune the GC to use more memory to reduce the number of garbage collections +export GOGC=400 +export DENDRITE_LINT_DISABLE_GC=1 + # Check that the servers build (this is done explicitly because `gb build` can silently fail (exit 0) and then we'd test a stale binary) gb build github.com/matrix-org/dendrite/cmd/dendrite-room-server gb build github.com/matrix-org/dendrite/cmd/roomserver-integration-tests @@ -12,8 +18,8 @@ gb build github.com/matrix-org/dendrite/cmd/dendrite-media-api-server gb build github.com/matrix-org/dendrite/cmd/mediaapi-integration-tests gb build github.com/matrix-org/dendrite/cmd/client-api-proxy -# Run the pre commit hooks -./hooks/pre-commit +# Run unit tests and linters +./scripts/build-test-lint.sh # Run the integration tests bin/roomserver-integration-tests