diff --git a/.github/workflows/benchmark-machine.yml b/.github/workflows/benchmark-machine.yml index 316cc55261..9ec848368d 100644 --- a/.github/workflows/benchmark-machine.yml +++ b/.github/workflows/benchmark-machine.yml @@ -9,21 +9,27 @@ on: required: true env: - INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }} # remote AWS host to run benchmarking + INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }} BENCHMARK_SSH_USER: ${{ secrets.BENCHMARK_SSH_USER }} - BENCHMARK_SSH_KEYPATH: ${{ secrets.BENCHMARK_SSH_KEYPATH }} + BENCHMARK_SSH_KEY: ${{ secrets.BENCHMARK_SSH_KEY }} jobs: ## run the benchmarking remotely benchmark-machine: - runs-on: jumphost + runs-on: ubuntu-latest steps: - name: Checkout codes on ${{ github.ref }} uses: actions/checkout@v4 with: fetch-depth: 0 - # TODO: maybe use GHA to start/stop remote instance + - name: Set up AWS CLI + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Start remote instance timeout-minutes: 10 id: start_instance @@ -38,26 +44,30 @@ jobs: break else sleep 20 + SECONDS=$((SECONDS + 20)) fi done echo "Remote instance reachable now after $SECONDS seconds" - remote_ip=`aws ec2 describe-instances --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text` - echo "Running instances ip address: $remote_ip" + remote_ip=$(aws ec2 describe-instances --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text) + echo "Running instances IP address: $remote_ip" echo "remote_ip=$remote_ip" >> $GITHUB_OUTPUT - # exit status should propagate through ssh - name: Remotely benchmark machine timeout-minutes: 10 run: | - echo "Running instances ip address: ${{ steps.start_instance.outputs.remote_ip }}" - ssh -x -o StrictHostKeychecking=no "${{ steps.start_instance.outputs.remote_ip }}" -l ${{ env.BENCHMARK_SSH_USER }} -i ${{ env.BENCHMARK_SSH_KEYPATH }} \ - docker pull litentry/litentry-parachain:${{ github.event.inputs.docker_tag }} && \ - docker run --rm litentry/litentry-parachain:${{ github.event.inputs.docker_tag }} benchmark machine --allow-fail --chain=litmus-dev + echo "Running instance's IP address: ${{ steps.start_instance.outputs.remote_ip }}" + cat << EOF > ./benchmark-server-key.pem + ${{ env.BENCHMARK_SSH_KEY }} + EOF + chmod 600 ./benchmark-server-key.pem + ssh -o StrictHostKeyChecking=no -i ./benchmark-server-key.pem "${{ env.BENCHMARK_SSH_USER }}@${{ steps.start_instance.outputs.remote_ip }}" \ + "docker pull litentry/litentry-parachain:${{ github.event.inputs.docker_tag }} && \ + docker run --rm litentry/litentry-parachain:${{ github.event.inputs.docker_tag }} benchmark machine --allow-fail --chain=litmus-dev" - name: Stop remote instance if: always() run: | aws ec2 stop-instances --instance-ids ${{ env.INSTANCE_ID }} sleep 5 - ret=`aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} | jq '.InstanceStatuses[0].InstanceState.Name'` + ret=$(aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].InstanceState.Name' --output text) echo "Remote instance running state: $ret" diff --git a/.github/workflows/benchmark-runtime-weights.yml b/.github/workflows/benchmark-runtime-weights.yml index 32e3461b8d..a9d6d85f0c 100644 --- a/.github/workflows/benchmark-runtime-weights.yml +++ b/.github/workflows/benchmark-runtime-weights.yml @@ -30,9 +30,9 @@ on: required: true env: - INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }} # remote AWS host to run benchmarking + INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }} BENCHMARK_SSH_USER: ${{ secrets.BENCHMARK_SSH_USER }} - BENCHMARK_SSH_KEYPATH: ${{ secrets.BENCHMARK_SSH_KEYPATH }} + BENCHMARK_SSH_KEY: ${{ secrets.BENCHMARK_SSH_KEY }} DOCKER_BUILDKIT: 1 jobs: @@ -77,7 +77,7 @@ jobs: ## run the benchmarking remotely benchmark: - runs-on: jumphost + runs-on: ubuntu-latest needs: build-docker # see https://github.com/actions/runner/issues/491 if: | @@ -111,25 +111,33 @@ jobs: run: | docker pull litentry/litentry-parachain:runtime-benchmarks + - name: Set up AWS CLI + uses: aws-actions/configure-aws-credentials@v2 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + # TODO: maybe use GHA to start/stop remote instance - name: Start remote instance timeout-minutes: 10 id: start_instance run: | - aws ec2 start-instances --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} + aws ec2 start-instances --instance-ids ${{ env.INSTANCE_ID }} sleep 5 - instance_status="aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].InstanceStatus.Status' --output text" - system_status="aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].SystemStatus.Status' --output text" + instance_status="aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].InstanceStatus.Status' --output text" + system_status="aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].SystemStatus.Status' --output text" SECONDS=0 while : ; do if [ "$(eval $instance_status)" = "ok" ] && [ "$(eval $system_status)" = "ok" ]; then break else sleep 20 + SECONDS=$((SECONDS + 20)) fi done echo "Remote instance reachable now after $SECONDS seconds" - remote_ip=`aws ec2 describe-instances --region ap-southeast-1 --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text` + remote_ip=`aws ec2 describe-instances --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text` echo "Running instances ip address: $remote_ip" echo "remote_ip=$remote_ip" >> $GITHUB_OUTPUT @@ -144,10 +152,14 @@ jobs: if [ "$arg" = "*" ]; then arg="\\$arg"; fi + cat << EOF > ./benchmark-server-key.pem + ${{ env.BENCHMARK_SSH_KEY }} + EOF + chmod 600 ./benchmark-server-key.pem for c in $chain; do - ssh -x -o StrictHostKeychecking=no ${{ secrets.BENCHMARK_INSTANCE_IP }} -l ${{ env.BENCHMARK_SSH_USER }} 'bash -s' < scripts/benchmark-weight-remote.sh "$c" "${GITHUB_REF#refs/heads/}" "$arg" + ssh -x -i ./benchmark-server-key.pem -o StrictHostKeychecking=no "${{ steps.start_instance.outputs.remote_ip }}" -l ${{ env.BENCHMARK_SSH_USER }} 'bash -s' < scripts/benchmark-weight-remote.sh "$c" "${GITHUB_REF#refs/heads/}" "$arg" echo "copy generated weights files back ..." - scp -o StrictHostKeychecking=no "${{ env.BENCHMARK_SSH_USER }}"@"${{ secrets.BENCHMARK_INSTANCE_IP }}":/tmp/litentry-parachain/runtime/$c/src/weights/*.rs runtime/$c/src/weights/ + scp -o StrictHostKeychecking=no -i ./benchmark-server-key.pem "${{ env.BENCHMARK_SSH_USER }}"@"${{ steps.start_instance.outputs.remote_ip }}":/tmp/litentry-parachain/runtime/$c/src/weights/*.rs runtime/$c/src/weights/ done echo "======================" git status @@ -155,9 +167,9 @@ jobs: - name: Stop remote instance if: always() run: | - aws ec2 stop-instances --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} + aws ec2 stop-instances --instance-ids ${{ env.INSTANCE_ID }} sleep 5 - ret=`aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} | jq '.InstanceStatuses[0].InstanceState.Name'` + ret=`aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} | jq '.InstanceStatuses[0].InstanceState.Name'` echo "Remote instance running state: $ret" - name: Create auto PR