diff --git a/.github/actions/build-library/action.yml b/.github/actions/build-library/action.yml new file mode 100644 index 0000000000..2c2a07d75a --- /dev/null +++ b/.github/actions/build-library/action.yml @@ -0,0 +1,111 @@ +# Configurable build step that build and test the library depending +name: 'Build and Test Library' +description: 'Runs the build and test steps of verifying ' +inputs: + scope: + description: > + The scope of the build. Must either be `full` or `minimal`. Full builds build for all + architectures and with LTO enabled. `minimal` only built for x86 and without LTO + required: true + default: 'minimal' + +runs: + using: "composite" + steps: + - name: Setup conditional environment variables + shell: | + if [[ $INPUT_SCOPE == full ]]; then + echo "BUILD_FLAGS=-PbuildTargetABIs=x86 -PenableLTO=false -PbuildCore=true" >> $GITHUB_ENV + echo "TEST_TARGET=connectedAndroidTest" >> $GITHUB_ENV + else + echo "BUILD_FLAGS=-PenableLTO=true -PbuildCore=true" >> $GITHUB_ENV + echo "TEST_TARGET=connectedObjectServerDebugAndroidTest" >> $GITHUB_ENV + endif + echo "CCACHE_CPP2=yes" >> $GITHUB_ENV + + # TODO Can we optimize this? Caching? + - name: Install dependencies + run: | + brew install ninja + brew install ccache + brew install yq + npm install -g mongodb-realm-cli + + # CCache setup copied from https://github.com/cristianadam/HelloWorld/blob/master/.github/workflows/build_cmake.yml + # TODO: How to verify this works at all? + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + shell: cmake -P {0} + run: | + string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) + message("::set-output name=timestamp::${current_date}") + + - name: ccache cache files + uses: actions/cache@v1.1.0 + with: + path: .ccache + key: ${{ runner.os }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + restore-keys: | + ${{ runner.os }}-ccache- + + # TODO: Figure out why we require 3.18.4. Ideally it should just be minimum versions everywhere? + - name: Setup cmake + uses: jwlawson/actions-setup-cmake@v1.8 + with: + cmake-version: '3.18.4' + + - name: Cache Gradle + uses: actions/cache@v2.1.0 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} + restore-keys: ${{ runner.os }}-gradle- + + - name: Setup SSH Keys and known_hosts + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: | + ssh-agent -a $SSH_AUTH_SOCK > /dev/null + ssh-add - <<< "${{ secrets.SSH_PRIVATE_KEY }}" + + # TODO: Figure out how to do different build flags depending on the build type. + - name: Build library + run: ./gradlew assemble $BUILD_FLAGS --stacktrace + + - name: Run Unit Tests + run: ./gradlew check $BUILD_FLAGS --stacktrace + + - name: Upload Unit Test Results + if: always() + uses: actions/upload-artifact@v2 + with: + name: unit-test-results + path: | + realm/realm-annotations-processor/build/test-results/test/TEST-*.xml + examples/unitTestExample/build/test-results/**/TEST-*.xml + realm/realm-library/build/test-results/**/TEST-*.xml + + - name: Run Integration Tests + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 29 + target: default + arch: x86_64 + ndk: 22.0.7026061 + profile: Nexus 6 + script: bash tools/sync_test_server/ci_run_integration_tests.sh + + # EnricoMi/publish-unit-test-result-action@v1 only works on Linux, so as a work-around + # we archive the test results from mac-os and download them to a Linux runner. + - name: Upload Integration Test Results + if: always() + uses: actions/upload-artifact@v2 + with: + name: integration-test-results + path: | + realm/realm-library/build/outputs/androidTest-results/connected/**/TEST-*.xml' + realm/kotlin-extensions/build/outputs/androidTest-results/connected/**/TEST-*.xml' + +# \ No newline at end of file diff --git a/.github/workflows/gradle-wrapper-validation.yml b/.github/workflows/gradle-wrapper-validation.yml index 405a2b3065..f365456a1e 100644 --- a/.github/workflows/gradle-wrapper-validation.yml +++ b/.github/workflows/gradle-wrapper-validation.yml @@ -1,5 +1,5 @@ name: "Validate Gradle Wrapper" -on: [push, pull_request] +on: [pull_request] jobs: validation: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000000..89fd9e5e6d --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,154 @@ +# Define how pull requests are built and tested. +name: Pull Requests +on: + pull_request: + paths-ignore: + - '*.md' + +env: + # GRADLE_OPTS: -Dorg.gradle.configureondemand=true -Dkotlin.incremental=false -Dorg.gradle.jvmargs="-Xmx4g -XX:MaxPermSize=4096m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 -XX:MaxMetaspaceSize=1024m" + + # GitHub Actions does not support variables. Use environments vars as a work-around + BUILD_FLAGS: -PbuildTargetABIs=x86 -PenableLTO=false -PbuildCore=true + + # TODO: From DockerFile, not sure if needed + CCACHE_CPP2: yes + # CMAKE_VERSION: 3.18.4 + # CCACHE_VERSION: 3.7.7 + +jobs: + run-pr-build: + runs-on: macos-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: 'recursive' + + # TODO Can we optimize this? Caching? + - name: Install dependencies + run: | + brew install ninja + brew install ccache + brew install yq + npm install -g mongodb-realm-cli + + # CCache setup copied from https://github.com/cristianadam/HelloWorld/blob/master/.github/workflows/build_cmake.yml + # TODO: How to verify this works at all? + - name: Prepare ccache timestamp + id: ccache_cache_timestamp + shell: cmake -P {0} + run: | + string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) + message("::set-output name=timestamp::${current_date}") + + - name: ccache cache files + uses: actions/cache@v1.1.0 + with: + path: .ccache + key: ${{ runner.os }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} + restore-keys: | + ${{ runner.os }}-ccache- + + # TODO: Figure out why we require 3.18.4. Ideally it should just be minimum versions everywhere? + - name: Setup cmake + uses: jwlawson/actions-setup-cmake@v1.8 + with: + cmake-version: '3.18.4' + + - name: Cache Gradle packages + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + + - name: Setup SSH Keys and known_hosts + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + run: | + ssh-agent -a $SSH_AUTH_SOCK > /dev/null + ssh-add - <<< "${{ secrets.SSH_PRIVATE_KEY }}" + + # TODO: Figure out how to do different build flags depending on the build type. + - name: Build library + run: ./gradlew assemble $BUILD_FLAGS --stacktrace + + - name: Run Unit Tests + run: ./gradlew check $BUILD_FLAGS --stacktrace + + - name: Upload Unit Test Results + if: always() + uses: actions/upload-artifact@v2 + with: + name: unit-test-results + if-no-files-found: warn + path: | + realm/realm-annotations-processor/build/test-results/test/TEST-*.xml + examples/unitTestExample/build/test-results/**/TEST-*.xml + realm/realm-library/build/test-results/**/TEST-*.xml + + - name: Run Integration Tests + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 29 + target: default + arch: x86_64 + ndk: 22.0.7026061 + profile: Nexus 6 + script: bash tools/sync_test_server/ci_run_integration_tests.sh + + # EnricoMi/publish-unit-test-result-action@v1 only works on Linux, so as a work-around + # we archive the test results from mac-os and download them to a Linux runner. + - name: Archive Integration Test Results and Logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: integration-test-results + if-no-files-found: warn + path: | + realm/realm-library/build/outputs/androidTest-results/connected/**/TEST-*.xml + realm/kotlin-extensions/build/outputs/androidTest-results/connected/**/TEST-*.xml + tools/sync_test_server/logcat.txt + tools/sync_test_server/baas_local/tmp-baas/baas_server.log + tools/sync_test_server/baas_local/tmp-command-server/command_server.log + + # Source: https://docs.github.com/en/actions/guides/building-and-testing-java-with-gradle + - name: Cleanup Gradle Cache + if: always() + # Remove some files from the Gradle cache, so they aren't cached by GitHub Actions. + # Restoring these files from a GitHub Actions cache might cause problems for future builds. + run: | + rm -f ~/.gradle/caches/modules-2/modules-2.lock + rm -f ~/.gradle/caches/modules-2/gc.properties + + +# TODO: Figure out another way to publish Unit Test Results. Maybe https://github.com/mikepenz/action-junit-report +# process-test-results: +# needs: run-pr-build +# runs-on: ubuntu-latest +# steps: +# - name: Download Artifacts +# uses: actions/download-artifact@v2 +# with: +# path: artifacts +# +# - name: Publish Annotation Processor Results +# uses: EnricoMi/publish-unit-test-result-action@master +# with: +# files: artifacts/unit-test-results/realm-java/realm/realm-annotations-processor/build/test-results/test/TEST-*.xml +# +# - name: Publish Example Project Results +# uses: EnricoMi/publish-unit-test-result-action@master +# with: +# files: artifacts/unit-test-results/realm-java/examples/unitTestExample/build/test-results/**/TEST-*.xml +# +# - name: Publish Library Test Results +# uses: EnricoMi/publish-unit-test-result-action@master +# with: +# files: artifacts/unit-test-results/realm-java/realm/realm-library/build/test-results/**/TEST-*.xml diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml new file mode 100644 index 0000000000..728b8a62ec --- /dev/null +++ b/.github/workflows/releases.yml @@ -0,0 +1,16 @@ +# Workflow for building and publishing releases to Maven Central. +name: Publish release +on: + push: + tags: + - 'v*' + +jobs: + run-release-build: + runs-on: macos-latest + steps: + - name: Build and Test Library + run: echo "Work work work..." + + + diff --git a/.github/workflows/snapshots.yml b/.github/workflows/snapshots.yml new file mode 100644 index 0000000000..9ea23c351c --- /dev/null +++ b/.github/workflows/snapshots.yml @@ -0,0 +1,17 @@ +# Workflow for building and publishing SNAPSHOT releases to Maven Central. +name: Create SNAPSHOT release +on: + push: + branches: + - master + - releases +env: +jobs: + run-snapshot-build: + runs-on: macos-latest + steps: + - name: Build and Test Library + run: echo "Work work work..." + + + diff --git a/.gitignore b/.gitignore index 8754b69638..95b2c86870 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,10 @@ realm/build # Gradle local properties local.properties +# Test server temp dirs +tools/sync_test_server/baas_local/tmp-baas +tools/sync_test_server/baas_local/tmp-command-server + # Core core core-* diff --git a/Jenkinsfile b/DEPRECATED-Jenkinsfile similarity index 100% rename from Jenkinsfile rename to DEPRECATED-Jenkinsfile diff --git a/build.gradle b/build.gradle index 7450372ee4..d29d972e5c 100644 --- a/build.gradle +++ b/build.gradle @@ -3,7 +3,7 @@ buildscript { jcenter() } dependencies { - classpath 'ch.netzwerg:gradle-release-plugin:1.2.0' + classpath 'ch.netzwerg:gradle-release-plugin:1.2.5' } } diff --git a/dependencies.list b/dependencies.list index 4c90a89d30..8575fa038b 100644 --- a/dependencies.list +++ b/dependencies.list @@ -2,9 +2,9 @@ # https://github.com/realm/realm-core/releases REALM_CORE=10.3.3 -# Version of MongoDB Realm used by integration tests -# See https://github.com/realm/ci/packages/147854 for available versions -MONGODB_REALM_SERVER=2021-03-09 +# Git commit SHA of MongoDB Realm used by integration tests +# See https://github.com/10gen/baas for available versions +MONGODB_REALM_SERVER=acb927b3b2042b4df38e00928183e62c29b20236 # Common Android settings across projects GRADLE_BUILD_TOOLS=4.0.0 diff --git a/realm/realm-library/src/syncIntegrationTest/kotlin/io/realm/SyncSessionTests.kt b/realm/realm-library/src/syncIntegrationTest/kotlin/io/realm/SyncSessionTests.kt index 311ccdacfd..852deef351 100644 --- a/realm/realm-library/src/syncIntegrationTest/kotlin/io/realm/SyncSessionTests.kt +++ b/realm/realm-library/src/syncIntegrationTest/kotlin/io/realm/SyncSessionTests.kt @@ -272,7 +272,7 @@ class SyncSessionTests { realm.executeTransaction { realm.createObject(SyncAllTypesWithFloat::class.java, ObjectId()) } - assertFailsWithErrorCode(ErrorCode.INVALID_SCHEMA_CHANGE) { + assertFailsWithErrorCode(ErrorCode.BAD_CHANGESET) { realm.syncSession.uploadAllLocalChanges() } } diff --git a/tools/sync_test_server/Dockerfile b/tools/sync_test_server/baas_docker/Dockerfile similarity index 96% rename from tools/sync_test_server/Dockerfile rename to tools/sync_test_server/baas_docker/Dockerfile index 1fda76b43c..434a15fd41 100644 --- a/tools/sync_test_server/Dockerfile +++ b/tools/sync_test_server/baas_docker/Dockerfile @@ -26,7 +26,7 @@ RUN apt-get update \ # Copy webserver script and install dependencies WORKDIR "/tmp" -COPY mongodb-realm-command-server.js /tmp +COPY ../mongodb-realm-command-server.js /tmp RUN npm install winston@2.4.0 temp httpdispatcher@1.0.0 fs-extra moment is-port-available@0.1.5 # Run integration test server diff --git a/tools/sync_test_server/start_server.sh b/tools/sync_test_server/baas_docker/start_docker_server.sh similarity index 97% rename from tools/sync_test_server/start_server.sh rename to tools/sync_test_server/baas_docker/start_docker_server.sh index 74962e3333..1ae4066b3b 100755 --- a/tools/sync_test_server/start_server.sh +++ b/tools/sync_test_server/baas_docker/start_docker_server.sh @@ -24,7 +24,7 @@ fi # Get the script dir which contains the Dockerfile DOCKERFILE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -MONGODB_REALM_VERSION=$(grep MONGODB_REALM_SERVER $DOCKERFILE_DIR/../../dependencies.list | cut -d'=' -f2) +MONGODB_REALM_VERSION=$(grep MONGODB_REALM_SERVER $DOCKERFILE_DIR/../../../dependencies.list | cut -d'=' -f2) adb reverse tcp:9443 tcp:9443 && \ adb reverse tcp:9080 tcp:9080 && \ diff --git a/tools/sync_test_server/stop_server.sh b/tools/sync_test_server/baas_docker/stop_docker_server.sh similarity index 100% rename from tools/sync_test_server/stop_server.sh rename to tools/sync_test_server/baas_docker/stop_docker_server.sh diff --git a/tools/sync_test_server/baas_local/abspath.sh b/tools/sync_test_server/baas_local/abspath.sh new file mode 100755 index 0000000000..4ab1dc323c --- /dev/null +++ b/tools/sync_test_server/baas_local/abspath.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +PLATFORM=$(uname -s) +case $(uname -s) in + Darwin) + exec perl -e 'use File::Spec; print File::Spec->rel2abs(shift); print "\n"' $1 + ;; + CYGWIN*) + exec cygpath -am $1 + ;; + *) + exec realpath -s $1 + ;; +esac \ No newline at end of file diff --git a/tools/sync_test_server/baas_local/install_baas.sh b/tools/sync_test_server/baas_local/install_baas.sh new file mode 100644 index 0000000000..fe1b6fff2e --- /dev/null +++ b/tools/sync_test_server/baas_local/install_baas.sh @@ -0,0 +1,264 @@ +#!/bin/bash +# Clone of https://raw.githubusercontent.com/realm/realm-core/master/evergreen/install_baas.sh +# But removed the part that import apps. +# +# This script will download all the dependencies for and build/start a Realm Cloud app server. +# +# Usage: +# ./evergreen/build_and_run_baas.sh {path to working directory} [git revision of baas] +# + +set -o errexit +set -o pipefail + +# Go and CCache on Mac does not play well together +unset -v CC + +case $(uname -s) in + Darwin) + STITCH_SUPPORT_LIB_URL="https://s3.amazonaws.com/stitch-artifacts/stitch-support/stitch-support-macos-debug-4.3.2-721-ge791a2e-patch-5e2a6ad2a4cf473ae2e67b09.tgz" + STITCH_ASSISTED_AGG_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_osx_ac073d06065af6e00103a8a3cf64672a3f875bea_20_12_01_19_47_16/assisted_agg" + GO_URL="https://golang.org/dl/go1.14.10.darwin-amd64.tar.gz" + MONGODB_DOWNLOAD_URL="http://downloads.10gen.com/osx/mongodb-macos-x86_64-enterprise-4.4.1.tgz" + YQ_DOWNLOAD_URL="https://github.com/mikefarah/yq/releases/download/3.4.1/yq_darwin_amd64" + JQ_DOWNLOAD_URL="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64" + ;; + Linux) + GO_URL="https://golang.org/dl/go1.14.10.linux-amd64.tar.gz" + YQ_DOWNLOAD_URL="https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64" + JQ_DOWNLOAD_URL="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" + + # Detect what distro/versionf of Linux we are running on to download the right version of MongoDB to download + # /etc/os-release covers debian/ubuntu/suse + if [[ -e /etc/os-release ]]; then + # Amazon Linux 2 comes back as 'amzn' + DISTRO_NAME="$(. /etc/os-release ; echo "$ID")" + DISTRO_VERSION="$(. /etc/os-release ; echo "$VERSION_ID")" + DISTRO_VERSION_MAJOR="$(cut -d. -f1 <<< "$DISTRO_VERSION" )" + elif [[ -e /etc/redhat-release ]]; then + # /etc/redhat-release covers RHEL + DISTRO_NAME=rhel + DISTRO_VERSION="$(lsb_release -s -r)" + DISTRO_VERSION_MAJOR="$(cut -d. -f1 <<< "$DISTRO_VERSION" )" + fi + case $DISTRO_NAME in + ubuntu) + MONGODB_DOWNLOAD_URL="http://downloads.10gen.com/linux/mongodb-linux-$(uname -m)-enterprise-ubuntu${DISTRO_VERSION_MAJOR}04-4.4.1.tgz" + STITCH_ASSISTED_AGG_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_ubuntu2004_x86_64_ac073d06065af6e00103a8a3cf64672a3f875bea_20_12_01_19_47_16/libmongo-ubuntu2004-x86_64.so" + STITCH_SUPPORT_LIB_URL="https://mciuploads.s3.amazonaws.com/mongodb-mongo-v4.4/stitch-support/ubuntu2004/03d22bb5884e280934d36702136d99a9363fb720/stitch-support-4.4.2-rc0-3-g03d22bb.tgz" + ;; + rhel) + case $DISTRO_VERSION_MAJOR in + 7) + MONGODB_DOWNLOAD_URL="http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-rhel70-4.4.1.tgz" + STITCH_ASSISTED_AGG_LIB_URL="https://stitch-artifacts.s3.amazonaws.com/stitch-mongo-libs/stitch_mongo_libs_linux_64_ac073d06065af6e00103a8a3cf64672a3f875bea_20_12_01_19_47_16/libmongo.so" + STITCH_SUPPORT_LIB_URL="https://s3.amazonaws.com/stitch-artifacts/stitch-support/stitch-support-rhel-70-4.3.2-721-ge791a2e-patch-5e2a6ad2a4cf473ae2e67b09.tgz" + ;; + *) + echo "Unsupported version of RHEL $DISTRO_VERSION" + exit 1 + ;; + esac + ;; + *) + echo "Unsupported platform $DISTRO_NAME $DISTRO_VERSION" + exit 1 + ;; + esac + ;; + *) + echo "Unsupported platform $(uname -s)" + exit 1 + ;; +esac + +# Allow path to $CURL to be overloaded by an environment variable +CURL=${CURL:=curl} + +BASE_PATH=$(cd $(dirname "$0"); pwd) + +REALPATH=$BASE_PATH/abspath.sh + +if [[ -z $1 ]]; then + echo "Must specify working directory." + exit 1 +fi +WORK_PATH=$($REALPATH $1) +BAAS_VERSION=$2 + +[[ -d $WORK_PATH ]] || mkdir -p $WORK_PATH +cd $WORK_PATH + +if [[ -f $WORK_PATH/baas_ready ]]; then + rm $WORK_PATH/baas_ready +fi + +echo "Installing node and go to build baas and its dependencies" +echo "Assume node is available" +#export NVM_DIR="$WORK_PATH/.nvm" +#if [ ! -d "$NVM_DIR" ]; then +# git clone https://github.com/nvm-sh/nvm.git "$NVM_DIR" +# cd "$NVM_DIR" +# git checkout `git describe --abbrev=0 --tags --match "v[0-9]*" $(git rev-list --tags --max-count=1)` +# cd - +#fi +#[[ -s "$NVM_DIR/nvm.sh" ]] && \. "$NVM_DIR/nvm.sh" +#NODE_VERSION=12.16.2 +#nvm install --no-progress $NODE_VERSION +#nvm use $NODE_VERSION + +echo "Assume Go is available" +#[[ -x $WORK_PATH/go/bin/go ]] || ($CURL -sL $GO_URL | tar -xz) +#export GOROOT=$WORK_PATH/go +#export PATH=$WORK_PATH/go/bin:$PATH + +echo "Assume jq and yg is available" +[[ -d baas_dep_binaries ]] || mkdir baas_dep_binaries +export PATH=$WORK_PATH/baas_dep_binaries:$PATH +#if [[ ! -x baas_dep_binaries/yq || ! -x baas_dep_binaries/jq ]]; then +# cd baas_dep_binaries +# which yq || ($CURL -LsS $YQ_DOWNLOAD_URL > yq && chmod +x yq) +# which jq || ($CURL -LsS $JQ_DOWNLOAD_URL > jq && chmod +x jq) +# cd - +#fi + +# If no baas version is specified. Find latest released version +if [[ -z "$BAAS_VERSION" ]]; then + BAAS_VERSION=$($CURL -LsS "https://realm.mongodb.com/api/private/v1.0/version" | jq -r '.backend.git_hash') +fi + +if [[ ! -d $WORK_PATH/baas/.git ]]; then + git clone git@github.com:10gen/baas.git + +fi + +cd baas +echo "Checking out baas version $BAAS_VERSION" +git checkout $BAAS_VERSION +cd - + +if [[ ! -d $WORK_PATH/baas/etc/dylib/lib ]]; then + echo "Downloading stitch support library" + mkdir baas/etc/dylib + cd baas/etc/dylib + $CURL -LsS $STITCH_SUPPORT_LIB_URL | tar -xz --strip-components=1 + cd - +fi +export LD_LIBRARY_PATH=$WORK_PATH/baas/etc/dylib/lib + +if [[ ! -x $WORK_PATH/baas_dep_binaries/libmongo.so && -n "$STITCH_ASSISTED_AGG_LIB_URL" ]]; then + echo "Downloading assisted agg library" + cd $WORK_PATH/baas_dep_binaries + $CURL -LsS $STITCH_ASSISTED_AGG_LIB_URL > libmongo.so + chmod 755 libmongo.so + cd - +fi + +if [[ ! -x $WORK_PATH/baas_dep_binaries/assisted_agg && -n "$STITCH_ASSISTED_AGG_URL" ]]; then + echo "Downloading assisted agg binary" + cd $WORK_PATH/baas_dep_binaries + $CURL -LsS $STITCH_ASSISTED_AGG_URL > assisted_agg + chmod 755 assisted_agg + cd - +fi + +YARN=$WORK_PATH/yarn/bin/yarn +if [[ ! -x $YARN ]]; then + echo "Getting yarn" + mkdir yarn && cd yarn + $CURL -LsS https://s3.amazonaws.com/stitch-artifacts/yarn/latest.tar.gz | tar -xz --strip-components=1 + cd - + mkdir $WORK_PATH/yarn_cache +fi + +if [[ ! -x baas_dep_binaries/transpiler ]]; then + echo "Building transpiler" + cd baas/etc/transpiler + $YARN --non-interactive --silent --cache-folder $WORK_PATH/yarn_cache + $YARN build --cache-folder $WORK_PATH/yarn_cache --non-interactive --silent + cd - + ln -s $(pwd)/baas/etc/transpiler/bin/transpiler baas_dep_binaries/transpiler +fi + +if [[ ! -x baas_dep_binaries/stitch-cli ]]; then + mkdir stitch-cli + cd stitch-cli + $CURL -LsS https://github.com/10gen/stitch-cli/archive/v1.9.0.tar.gz | tar -xz --strip-components=1 + go build -o $WORK_PATH/baas_dep_binaries/stitch-cli + cd - +fi + +if [ ! -x $WORK_PATH/mongodb-binaries/bin/mongod ]; then + echo "Downloading mongodb" + $CURL -sLS $MONGODB_DOWNLOAD_URL --output mongodb-binaries.tgz + + tar -xzf mongodb-binaries.tgz + rm mongodb-binaries.tgz + mv mongodb* mongodb-binaries + chmod +x ./mongodb-binaries/bin/* +fi + +if [[ -d mongodb-dbpath ]]; then + rm -rf mongodb-dbpath +fi +mkdir mongodb-dbpath + +function cleanup() { + if [[ -f $WORK_PATH/baas_server.pid ]]; then + PIDS_TO_KILL="$(< $WORK_PATH/baas_server.pid)" + fi + + if [[ -f $WORK_PATH/mongod.pid ]]; then + PIDS_TO_KILL="$(< $WORK_PATH/mongod.pid) $PIDS_TO_KILL" + fi + + if [[ -n "$PIDS_TO_KILL" ]]; then + echo "Killing $PIDS_TO_KILL" + kill $PIDS_TO_KILL + echo "Waiting for processes to exit" + wait + fi +} + +trap "exit" INT TERM ERR +trap cleanup EXIT + +echo "Starting mongodb" +[[ -f $WORK_PATH/mongodb-dbpath/mongod.pid ]] && rm $WORK_PATH/mongodb-path/mongod.pid +./mongodb-binaries/bin/mongod \ + --replSet rs \ + --bind_ip_all \ + --port 26000 \ + --logpath $WORK_PATH/mongodb-dbpath/mongod.log \ + --dbpath $WORK_PATH/mongodb-dbpath/ \ + --pidfilepath $WORK_PATH/mongod.pid & + +./mongodb-binaries/bin/mongo \ + --nodb \ + --eval 'assert.soon(function(x){try{var d = new Mongo("localhost:26000"); return true}catch(e){return false}}, "timed out connecting")' \ +> /dev/null + +echo "Initializing replica set" +./mongodb-binaries/bin/mongo --port 26000 --eval 'rs.initiate()' > /dev/null + +cd $WORK_PATH/baas +echo "Adding stitch user" +go run -exec="env LD_LIBRARY_PATH=$LD_LIBRARY_PATH" cmd/auth/user.go \ + addUser \ + -domainID 000000000000000000000000 \ + -mongoURI mongodb://localhost:26000 \ + -salt 'DQOWene1723baqD!_@#'\ + -id "unique_user@domain.com" \ + -password "password" + +[[ -d tmp ]] || mkdir tmp +echo "Starting stitch app server" +[[ -f $WORK_PATH/baas_server.pid ]] && rm $WORK_PATH/baas_server.pid +go build -o $WORK_PATH/baas_server cmd/server/main.go +$WORK_PATH/baas_server \ + --configFile=etc/configs/test_config.json 2>&1 > $WORK_PATH/baas_server.log & +echo $! > $WORK_PATH/baas_server.pid +$BASE_PATH/wait_for_baas.sh $WORK_PATH/baas_server.pid +touch $WORK_PATH/baas_ready +echo "Baas server ready" +wait diff --git a/tools/sync_test_server/baas_local/start_local_server.sh b/tools/sync_test_server/baas_local/start_local_server.sh new file mode 100755 index 0000000000..43e7163e60 --- /dev/null +++ b/tools/sync_test_server/baas_local/start_local_server.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# +# This script controls starting a MongoDB Realm test server and import two running apps into it. +# +# It also starts a local command server that the integration tests use to control the MongoDB Realm +# instance. +# +# +# Copy from https://github.com/realm/ci/edit/master/realm/docker/mongodb-realm/run.sh +# Requires jq and mongodb-realm-cli + +set -e + +function import_apps () { + app_dir="$1" + realm-cli login --config-path=/tmp/stitch-config --base-url=http://localhost:9090 --auth-provider=local-userpass --username=unique_user@domain.com --password=password + access_token=$(yq e ".access_token" /tmp/stitch-config) + group_id=$(curl --header "Authorization: Bearer $access_token" http://localhost:9090/api/admin/v3.0/auth/profile -s | jq '.roles[0].group_id' -r) + cd $app_dir + for app in *; do + echo "importing app: ${app}" + + manifest_file="config.json" + app_id_param="" + if [ -f "$app/secrets.json" ]; then + # create app by importing an empty skeleton with the same name + app_name=$(jq '.name' "$app/$manifest_file" -r) + temp_app="/tmp/stitch-apps/$app" + mkdir -p "$temp_app" && echo "{ \"name\": \"$app_name\" }" > "$temp_app/$manifest_file" + realm-cli import --config-path=/tmp/stitch-config --base-url=http://localhost:9090 --path="$temp_app" --project-id $group_id -y --strategy replace + + app_id=$(jq '.app_id' "$temp_app/$manifest_file" -r) + app_id_param="--app-id=$app_id" + + # import secrets into the created app + while read -r secret value; do + realm-cli secrets add --config-path=/tmp/stitch-config --base-url=http://localhost:9090 --app-id=$app_id --name="$secret" --value="$(echo $value | sed 's/\\n/\n/g')" + done < <(jq 'to_entries[] | [.key, .value] | @tsv' "$app/secrets.json" -r) + fi + + realm-cli import --config-path=/tmp/stitch-config --base-url=http://localhost:9090 --path="$app" $app_id_param --project-id $group_id -y --strategy replace + jq '.app_id' "$app/$manifest_file" -r > "$app/app_id" + done +} + +SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +MONGODB_REALM_VERSION=$(grep MONGODB_REALM_SERVER $SCRIPTPATH/../../../dependencies.list | cut -d'=' -f2) + +# Running as root is not directly required here, but we might want to access files later on the +# emulator. Set root here to avoid resetting network settings later. +adb root +adb reverse tcp:9443 tcp:9443 && \ +adb reverse tcp:9080 tcp:9080 && \ +adb reverse tcp:9090 tcp:9090 && \ +adb reverse tcp:8888 tcp:8888 || { echo "Failed to reverse adb port." ; exit 1 ; } + +SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +# Cleanup any old installation +echo "Cleanup old tmp directories..." +if [[ -d tmp-command-server ]]; then + rm -rf tmp-command-server +fi +if [[ -d tmp-baas ]]; then + rm -rf tmp-baas +fi + +# Install and run BAAS +echo "Install and start BAAS: $MONGODB_REALM_VERSION ..." +sh ./install_baas.sh ./tmp-baas $MONGODB_REALM_VERSION & +while [[ ! -e $SCRIPTPATH/tmp-baas/baas_ready ]] ; do + sleep 1 +done + +# Create app configurations +echo "Prepare app templates..." +APP_CONFIG_DIR=`mktemp -d -t app_config` +$SCRIPTPATH/../app_config_generator.sh $APP_CONFIG_DIR $SCRIPTPATH/../app_template testapp1 testapp2 + +echo "Import apps..." +import_apps "$APP_CONFIG_DIR" +cd $SCRIPTPATH + +# Start command server +echo "Start command server..." +mkdir tmp-command-server || true +cd tmp-command-server +cp $SCRIPTPATH/../mongodb-realm-command-server.js ./ +npm install winston@2.4.0 temp httpdispatcher@1.0.0 fs-extra moment is-port-available@0.1.5 +node ./mongodb-realm-command-server.js $APP_CONFIG_DIR > command_server.log & +echo $! > command_server.pid diff --git a/tools/sync_test_server/baas_local/stop_local_server.sh b/tools/sync_test_server/baas_local/stop_local_server.sh new file mode 100755 index 0000000000..98f08462c4 --- /dev/null +++ b/tools/sync_test_server/baas_local/stop_local_server.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# +# Stops any running BAAS and command server +# +SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +if [[ -f $SCRIPTPATH/tmp-command-server/command_server.pid ]]; then + PIDS_TO_KILL="$(< $SCRIPTPATH/tmp-command-server/command_server.pid)" +fi +if [[ -f $SCRIPTPATH/tmp-baas/baas_server.pid ]]; then + PIDS_TO_KILL="$(< $SCRIPTPATH/tmp-baas/baas_server.pid) $PIDS_TO_KILL" +fi +if [[ -f $SCRIPTPATH/tmp-baas/mongod.pid ]]; then + PIDS_TO_KILL="$(< $SCRIPTPATH/tmp-baas/mongod.pid) $PIDS_TO_KILL" +fi +if [[ -f $SCRIPTPATH/tmp-baas/baas_ready ]]; then + rm $SCRIPTPATH/tmp-baas/baas_ready +fi +if [[ -n "$PIDS_TO_KILL" ]]; then + echo "Killing $PIDS_TO_KILL" + kill $PIDS_TO_KILL +fi \ No newline at end of file diff --git a/tools/sync_test_server/baas_local/wait_for_baas.sh b/tools/sync_test_server/baas_local/wait_for_baas.sh new file mode 100755 index 0000000000..604ae5bb1d --- /dev/null +++ b/tools/sync_test_server/baas_local/wait_for_baas.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -o errexit +set -o pipefail + +CURL=${CURL:=curl} +STITCH_PID_FILE=$1 +RETRY_COUNT=${2:-36} + +WAIT_COUNTER=0 +until $CURL --output /dev/null --head --fail http://localhost:9090 --silent ; do + if [[ -f $STITCH_PID_FILE ]]; then + pgrep -F $STITCH_PID_FILE > /dev/null || (echo "Stitch $(< $STITCH_PID_FILE) is not running"; exit 1) + fi + + WAIT_COUNTER=$(($WAIT_COUNTER + 1 )) + if [[ $WAIT_COUNTER = $RETRY_COUNT ]]; then + echo "Timed out waiting for stitch to start" + exit 1 + fi + + sleep 5 +done diff --git a/tools/sync_test_server/ci_run_integration_tests.sh b/tools/sync_test_server/ci_run_integration_tests.sh new file mode 100755 index 0000000000..39bb9c8678 --- /dev/null +++ b/tools/sync_test_server/ci_run_integration_tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Script for running integration tests on CI +SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +echo "Start building local BAAS" +cd $SCRIPTPATH/baas_local +bash ./start_local_server.sh +cd ../../../realm +adb logcat -b all -c +adb logcat -v time > $SCRIPTPATH/logcat.txt & +LOG_CAT_PID=`echo \$!` +# TODO: Build parameters should be command line parameters +./gradlew connectedObjectServerDebugAndroidTest -PbuildTargetABIs=x86 -PenableLTO=false -PbuildCore=true +EXIT_CODE=`echo $?` +echo "Android tests exit code: $EXIT_CODE" +kill $LOG_CAT_PID +cd $SCRIPTPATH/baas_local +bash ./stop_local_server.sh +echo "Local servers stopped" +exit $EXIT_CODE \ No newline at end of file diff --git a/tools/sync_test_server/mongodb-realm-command-server.js b/tools/sync_test_server/mongodb-realm-command-server.js index d38c271cde..fe454d0136 100755 --- a/tools/sync_test_server/mongodb-realm-command-server.js +++ b/tools/sync_test_server/mongodb-realm-command-server.js @@ -10,6 +10,8 @@ * stopped, so a new integration test will start from a clean slate. */ +var commandlineArgs = process.argv.slice(2); +var APP_DIR = commandlineArgs[0] ? commandlineArgs[0] : "/apps"; var winston = require('winston'); //logging var http = require('http'); const fs = require('fs') @@ -44,7 +46,7 @@ function handleApplicationId(appName, req, resp) { switch(req.method) { case "GET": try { - const data = fs.readFileSync('/apps/' + appName + '/app_id', 'utf8') + const data = fs.readFileSync(APP_DIR + '/' + appName + '/app_id', 'utf8') console.log(data) resp.writeHead(200, {'Content-Type': 'text/plain'}); resp.end(data.replace(/\n$/, ''));