Skip to content

Commit

Permalink
ci: run benchmark on GitHub Actions
Browse files Browse the repository at this point in the history
  • Loading branch information
guitarrapc committed Jul 2, 2024
1 parent 7d2cdce commit 249746f
Show file tree
Hide file tree
Showing 3 changed files with 248 additions and 23 deletions.
56 changes: 56 additions & 0 deletions .github/scripts/dotnet_install.sh
Original file line number Diff line number Diff line change
@@ -0,0 1,56 @@
#!/bin/bash
set -euo pipefail

# Install .NET SDK over ssh
#
# Sample usage:
# $ ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/id_ed25519 [email protected] 'bash -s -- --dotnet-version 8.0' < ./scripts/dotnet_install.sh
# $ ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/id_ed25519 [email protected] 'bash -s -- --dotnet-version 8.0' < ./scripts/dotnet_install.sh
# $ echo $?
#

function usage {
echo "usage: $(basename $0) [options]"
echo "Options:"
echo " --dotnet-version string Version of dotnet sdk to install (default: 8.0)"
echo " --help Show this help message"
echo ""
echo "Examples:"
echo " 1. Install dotnet sdk version 8.0 over ssh"
echo " ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/id_ed25519 [email protected] 'bash -s -- --dotnet-version 8.0' < ./scripts/$(basename $0).sh"
echo ' echo $? # use $? to get the exit code of the remote command'
}

while [ $# -gt 0 ]; do
case $1 in
# optional
--dotnet-version) _DOTNET_VERSION=$2; shift 2; ;;
--help) usage; exit 1; ;;
*) shift ;;
esac
done

function print() {
echo ""
echo "$*"
}

dotnet_version="${_DOTNET_VERSION:="8.0"}"

# show machine name
print "MACHINE_NAME: $(hostname)"

# install dotnet (dotnet-install.sh must be downloaded before running script)
print "# Install dotnet sdk version: ${dotnet_version}"
sudo bash /opt/dotnet-install.sh --channel "${dotnet_version}" --install-dir /usr/share/dotnet

# link dotnet to /usr/local/bin
print "# Link to /usr/local/bin/dotnet"
if [[ ! -h "/usr/local/bin/dotnet" ]]; then
sudo ln -s /usr/share/dotnet/dotnet /usr/local/bin/dotnet
fi

# show dotnet verison
print "# Show installed dotnet sdk versions"
echo "dotnet sdk versions (list): $(dotnet --list-sdks)"
echo "dotnet sdk version (default): $(dotnet --version)"
133 changes: 133 additions & 0 deletions .github/scripts/run_benchmark.sh
Original file line number Diff line number Diff line change
@@ -0,0 1,133 @@
#!/bin/bash
set -euo pipefail

# git clone and run benchmark over ssh
#
# MagicOnion Server
# $ ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/id_ed25519 [email protected] 'bash -s -- --branch main --build-csproj "perf/BenchmarkApp/PerformanceTest.Server/PerformanceTest.Server.csproj" --env-settings "Kestrel__EndpointDefaults__Protocols=Http2;Kestrel__Endpoints__Grpc__Url=http:// :5000;" --owner Cysharp --repo MagicOnion' < ./scripts/run_benchmark.sh
# $ echo $?
#
# MagicOnion Client
# $ ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/id_ed25519 [email protected] 'bash -s -- --args "-u http://benchmark-server-vm:5000 -s streaminghub --channels 1 --streams 1" --branch main --build-csproj "perf/BenchmarkApp/PerformanceTest.Client/PerformanceTest.Client.csproj" --owner Cysharp --repo MagicOnion' < ./scripts/run_benchmark.sh
# $ echo $?

function usage {
echo "usage: $(basename $0) --build-csproj <string> --repo <string> [options]"
echo "Required:"
echo " --build-csproj string Path to the csproj file to build"
echo " --repo string Repository name to clone"
echo "Options:"
echo " --args string Arguments to pass when running the built binary (default: \"\")"
echo " --branch string Branch name to checkout (default: main)"
echo " --build-config string Build configuration (default: Release)"
echo " --env-settings string Environment settings to set before running the binary, use ; to delinate multiple environments (default: \"\")"
echo " --owner string Repository owner (default: Cysharp)"
echo " --help Show this help message"
echo ""
echo "Examples:"
echo " 1. Run MagicOnion Benchmark Server over ssh"
echo " ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/id_ed25519 [email protected] 'bash -s -- --branch main --build-csproj 'perf/BenchmarkApp/PerformanceTest.Server/PerformanceTest.Server.csproj' --env-settings \"Kestrel__EndpointDefaults__Protocols=Http2;Kestrel__Endpoints__Grpc__Url=http:// :5000;\" --owner Cysharp --repo MagicOnion' < ./scripts/$(basename $0).sh"
echo ' echo $? # use $? to get the exit code of the remote command'
echo ""
echo " 2. Run MagicOnion Benchmark Client over ssh"
echo " ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/id_ed25519 [email protected] 'bash -s -- --args \"-u http://benchmark-server-vm:5000 -s streaminghub --channels 1 --streams 1\" --branch main --build-csproj 'perf/BenchmarkApp/PerformanceTest.Client/PerformanceTest.Client.csproj' --owner Cysharp --repo MagicOnion' < ./scripts/$(basename $0).sh"
echo ' echo $? # use $? to get the exit code of the remote command'
}

while [ $# -gt 0 ]; do
case $1 in
# required
--build-csproj) _BUILD_CSPROJ=$2; shift 2; ;;
--repo) _REPO=$2; shift 2; ;;
# optional
--args) _ARGS=$2; shift 2; ;;
--branch) _BRANCH=$2; shift 2; ;;
--build-config) _BUILD_CONFIG=$2; shift 2; ;;
--env-settings) _ENV_SETTINGS=$2; shift 2; ;;
--owner) _OWNER=$2; shift 2; ;;
--help) usage; exit 1; ;;
*) shift ;;
esac
done

function print() {
echo ""
echo "$*"
}

# parameter setup
args="${_ARGS:=""}"
owner="${_OWNER:="Cysharp"}"
repo="${_REPO}"
branch="${_BRANCH:="main"}"
env_settings="${_ENV_SETTINGS:=""}"
build_config="${_BUILD_CONFIG:="Release"}"
build_csproj="${_BUILD_CSPROJ}"

binary_name=$(basename "$(dirname "$build_csproj")")
publish_dir="artifacts/$binary_name"
clone_path="$HOME/github/$repo"
full_process_path="$clone_path/$publish_dir/$binary_name"

# show machine name
print "MACHINE_NAME: $(hostname)"

# is dotnet installed?
print "# Show installed dotnet sdk versions"
echo "dotnet sdk versions (list): $(dotnet --list-sdks)"
echo "dotnet sdk version (default): $(dotnet --version)"

# setup env
print "# Setup environment"
IFS=';' read -ra env_array <<< "$env_settings"
for item in "${env_array[@]}"; do
if [ -n "$item" ]; then
export "$item"
fi
done
export

# git clone cysharp repo
print "# git clone $owner/$repo"
mkdir -p "$(dirname "$clone_path")"
if [[ ! -d "$clone_path" ]]; then
git clone "https://github.com/$owner/$repo" "$clone_path"
fi

# list files
print "# List cloned files"
ls "$clone_path"

# git pull
print "# git pull $branch"
pushd "$clone_path"
git switch "$branch"
git pull
git reset --hard HEAD
popd

# dotnet publish
print "# dotnet publish $build_csproj"
pushd "$clone_path"
print " ## list current files under $(pwd)"
ls -l

print " ## dotnet publish $build_csproj"
dotnet publish -c "$build_config" -p:PublishSingleFile=true --runtime linux-x64 --self-contained false "$build_csproj" -o "$publish_dir"

print " ## list published files under $publish_dir"
ls "$publish_dir"

print " ## add x permission to published file $full_process_path"
chmod x "$full_process_path"
popd

# process check
print "# Checking process $binary_name already runnning, kill if exists"
if pgrep -f "$full_process_path"; then
pkill -f "$full_process_path" || true
fi

# run dotnet app
print "# Run $full_process_path"
"$full_process_path" $args
82 changes: 59 additions & 23 deletions .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,36 3,72 @@ name: Benchmark
on:
push:
branches:
- feature/perf
- benchmark

env:
BUILD_CONFIG: Release

permissions:
contents: read
id-token: write

jobs:
build:
benchmark:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: Cysharp/Actions/.github/actions/setup-dotnet@main
# build
- name: Build Client
run: dotnet publish -c ${{ env.BUILD_CONFIG }} -p:PublishSingleFile=true --runtime linux-x64 --self-contained true ./perf/BenchmarkApp/PerformanceTest.Client/PerformanceTest.Client.csproj -o ./perf/artifacts/PerformanceTest.Client
- name: Build Server
run: dotnet publish -c ${{ env.BUILD_CONFIG }} -p:PublishSingleFile=true --runtime linux-x64 --self-contained true ./perf/BenchmarkApp/PerformanceTest.Server/PerformanceTest.Server.csproj -o ./perf/artifacts/PerformanceTest.Server
# upload
- uses: Cysharp/Actions/.github/actions/upload-artifact@main
- name: Load secret
id: op-load-secret
uses: 1password/load-secrets-action@v2
with:
name: artifacts
path: ./perf/artifacts
if-no-files-found: error
retention-days: 1

# run:
# needs: [build]
# runs-on: ubuntu-latest
# timeout-minutes: 10
# steps:
# - uses: actions/checkout@v4
# # run
# # get result
export-env: false
env:
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN_PUBLIC }}
SSH_KEY: "op://GitHubActionsPublic/BENCHMARK_VM_SSH/private key"
AZURE_OIDC_CLIENTID: "op://GitHubActionsPublic/BENCHMARK_AZURE_OIDC/clientId"
AZURE_OIDC_TENANTID: "op://GitHubActionsPublic/BENCHMARK_AZURE_OIDC/tenantId"
AZURE_OIDC_SUBSCRIPTIONID: "op://GitHubActionsPublic/BENCHMARK_AZURE_OIDC/subscriptionId"
- name: Output SSH Key
run: |
mkdir -p ~/.ssh
echo "${{ steps.op-load-secret.outputs.SSH_KEY }}" > ~/.ssh/id_ed25519
chmod 600 ~/.ssh/id_ed25519
- uses: azure/login@v2
with:
client-id: ${{ steps.op-load-secret.outputs.AZURE_OIDC_CLIENTID }}
tenant-id: ${{ steps.op-load-secret.outputs.AZURE_OIDC_TENANTID }}
subscription-id: ${{ steps.op-load-secret.outputs.AZURE_OIDC_SUBSCRIPTIONID }}
- name: Start Benchmark VM
run: |
for vm in $(az vm list -d -g BenchmarkTest --query "[?powerState!='VM running'].name" -o tsv); do
az vm start --name $vm --resource-group BenchmarkTest
done
- name: Setup .NET on Benchmark VM
run: |
for ip in $(az vm list -d -g BenchmarkTest --query "[?powerState=='VM running'].publicIps" -o tsv); do
echo "::group::Install on $ip"
ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/id_ed25519 azure-user@$ip 'bash -s -- --dotnet-version 8.0' < ./scripts/dotnet_install.sh
echo "::endgroup::"
done
working-directory: ./.github
- name: Run Benchmark Server
run: |
for ip in $(az vm list -d -g BenchmarkTest --query "[?powerState=='VM running' && name=='benchmark-server-vm'].publicIps" -o tsv); do
ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/id_ed25519 azure-user@$ip 'bash -s -- --branch main --build-csproj "perf/BenchmarkApp/PerformanceTest.Server/PerformanceTest.Server.csproj" --env-settings "Kestrel__EndpointDefaults__Protocols=Http2;Kestrel__Endpoints__Grpc__Url=http:// :5000;" --owner Cysharp --repo MagicOnion' < ./scripts/run_benchmark.sh &
done
# wait until server started
sleep 120s
working-directory: ./.github
- name: Run Benchmark Client
run: |
for ip in $(az vm list -d -g BenchmarkTest --query "[?powerState=='VM running' && name=='benchmark-client-vm'].publicIps" -o tsv); do
ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/id_ed25519 azure-user@$ip 'bash -s -- --args "-u http://benchmark-server-vm:5000 -s streaminghub --channels 1 --streams 1" --branch main --build-csproj "perf/BenchmarkApp/PerformanceTest.Client/PerformanceTest.Client.csproj" --owner Cysharp --repo MagicOnion' < ./scripts/run_benchmark.sh
done
working-directory: ./.github
- name: Stop (Deallocate) Benchmark VM
if: always()
run: |
for vm in $(az vm list -d -g BenchmarkTest --query "[?powerState=='VM running'].name" -o tsv); do
az vm deallocate --name $vm --resource-group BenchmarkTest --no-wait
done

0 comments on commit 249746f

Please sign in to comment.