Merge branch 'dev' into p-893-enhance-bitarcoss-worker-ceremony-proce…

…ssing-with-multithreading
litentry · Jul 18, 2024 · 4f25e18 · 4f25e18
2 parents 0232ad3 + 31bcf9b
commit 4f25e18
Show file tree

Hide file tree

Showing 23 changed files with 496 additions and 56 deletions.
diff --git a/.github/workflows/benchmark-machine.yml b/.github/workflows/benchmark-machine.yml
@@ -9,21 +9,27 @@ on:
         required: true
 
 env:
-  INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }}  # remote AWS host to run benchmarking
+  INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }}
   BENCHMARK_SSH_USER: ${{ secrets.BENCHMARK_SSH_USER }}
-  BENCHMARK_SSH_KEYPATH: ${{ secrets.BENCHMARK_SSH_KEYPATH }}
+  BENCHMARK_SSH_KEY: ${{ secrets.BENCHMARK_SSH_KEY }}
 
 jobs:
   ## run the benchmarking remotely
   benchmark-machine:
-    runs-on: jumphost
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout codes on ${{ github.ref }}
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      # TODO: maybe use GHA to start/stop remote instance
+      - name: Set up AWS CLI
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
       - name: Start remote instance
         timeout-minutes: 10
         id: start_instance
@@ -38,26 +44,30 @@ jobs:
               break
             else
               sleep 20
+              SECONDS=$((SECONDS + 20))
             fi
           done
           echo "Remote instance reachable now after $SECONDS seconds"
-          remote_ip=`aws ec2 describe-instances --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text`
-          echo "Running instances ip address: $remote_ip"
+          remote_ip=$(aws ec2 describe-instances --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text)
+          echo "Running instances IP address: $remote_ip"
           echo "remote_ip=$remote_ip" >> $GITHUB_OUTPUT
 
-      # exit status should propagate through ssh
       - name: Remotely benchmark machine
         timeout-minutes: 10
         run: |
-          echo "Running instances ip address: ${{ steps.start_instance.outputs.remote_ip }}"
-          ssh -x -o StrictHostKeychecking=no "${{ steps.start_instance.outputs.remote_ip }}" -l ${{ env.BENCHMARK_SSH_USER }} -i ${{ env.BENCHMARK_SSH_KEYPATH }} \
-            docker pull litentry/litentry-parachain:${{ github.event.inputs.docker_tag }} && \
-            docker run --rm litentry/litentry-parachain:${{ github.event.inputs.docker_tag }} benchmark machine --allow-fail --chain=litmus-dev
+          echo "Running instance's IP address: ${{ steps.start_instance.outputs.remote_ip }}"
+          cat << EOF > ./benchmark-server-key.pem
+          ${{ env.BENCHMARK_SSH_KEY }}
+          EOF
+          chmod 600 ./benchmark-server-key.pem
+          ssh -o StrictHostKeyChecking=no -i ./benchmark-server-key.pem "${{ env.BENCHMARK_SSH_USER }}@${{ steps.start_instance.outputs.remote_ip }}" \
+            "docker pull litentry/litentry-parachain:${{ github.event.inputs.docker_tag }} && \
+            docker run --rm litentry/litentry-parachain:${{ github.event.inputs.docker_tag }} benchmark machine --allow-fail --chain=litmus-dev"
 
       - name: Stop remote instance
         if: always()
         run: |
           aws ec2 stop-instances --instance-ids ${{ env.INSTANCE_ID }}
           sleep 5
-          ret=`aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} | jq '.InstanceStatuses[0].InstanceState.Name'`
+          ret=$(aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].InstanceState.Name' --output text)
           echo "Remote instance running state: $ret"
diff --git a/.github/workflows/benchmark-runtime-weights.yml b/.github/workflows/benchmark-runtime-weights.yml
@@ -30,9 +30,9 @@ on:
         required: true
 
 env:
-  INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }} # remote AWS host to run benchmarking
+  INSTANCE_ID: ${{ secrets.BENCHMARK_INSTANCE_ID }}
   BENCHMARK_SSH_USER: ${{ secrets.BENCHMARK_SSH_USER }}
-  BENCHMARK_SSH_KEYPATH: ${{ secrets.BENCHMARK_SSH_KEYPATH }}
+  BENCHMARK_SSH_KEY: ${{ secrets.BENCHMARK_SSH_KEY }}
   DOCKER_BUILDKIT: 1
 
 jobs:
@@ -77,7 +77,7 @@ jobs:
 
   ## run the benchmarking remotely
   benchmark:
-    runs-on: jumphost
+    runs-on: ubuntu-latest
     needs: build-docker
     # see https://github.com/actions/runner/issues/491
     if: |
@@ -111,25 +111,33 @@ jobs:
         run: |
           docker pull litentry/litentry-parachain:runtime-benchmarks
 
+      - name: Set up AWS CLI
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
       # TODO: maybe use GHA to start/stop remote instance
       - name: Start remote instance
         timeout-minutes: 10
         id: start_instance
         run: |
-          aws ec2 start-instances --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }}
+          aws ec2 start-instances --instance-ids ${{ env.INSTANCE_ID }}
           sleep 5
-          instance_status="aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].InstanceStatus.Status' --output text"
-          system_status="aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].SystemStatus.Status' --output text"
+          instance_status="aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].InstanceStatus.Status' --output text"
+          system_status="aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} --query 'InstanceStatuses[0].SystemStatus.Status' --output text"
           SECONDS=0
           while : ; do
             if [ "$(eval $instance_status)" = "ok" ] && [ "$(eval $system_status)" = "ok" ]; then
               break
             else
               sleep 20
+              SECONDS=$((SECONDS + 20))
             fi
           done
           echo "Remote instance reachable now after $SECONDS seconds"
-          remote_ip=`aws ec2 describe-instances --region ap-southeast-1 --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text`
+          remote_ip=`aws ec2 describe-instances --filters 'Name=instance-state-name,Values=running' 'Name=instance-id,Values=${{ env.INSTANCE_ID }}' --query 'Reservations[*].Instances[*].[PublicIpAddress]' --output text`
           echo "Running instances ip address: $remote_ip"
           echo "remote_ip=$remote_ip" >> $GITHUB_OUTPUT
 
@@ -144,20 +152,24 @@ jobs:
           if [ "$arg" = "*" ]; then
             arg="\\$arg";
           fi
+          cat << EOF > ./benchmark-server-key.pem
+          ${{ env.BENCHMARK_SSH_KEY }}
+          EOF
+          chmod 600 ./benchmark-server-key.pem
           for c in $chain; do
-            ssh -x -o StrictHostKeychecking=no ${{ secrets.BENCHMARK_INSTANCE_IP }} -l ${{ env.BENCHMARK_SSH_USER }} 'bash -s' < scripts/benchmark-weight-remote.sh "$c" "${GITHUB_REF#refs/heads/}" "$arg"
+            ssh -x -i ./benchmark-server-key.pem -o StrictHostKeychecking=no "${{ steps.start_instance.outputs.remote_ip }}" -l ${{ env.BENCHMARK_SSH_USER }} 'bash -s' < scripts/benchmark-weight-remote.sh "$c" "${GITHUB_REF#refs/heads/}" "$arg"
             echo "copy generated weights files back ..."
-            scp -o StrictHostKeychecking=no "${{ env.BENCHMARK_SSH_USER }}"@"${{ secrets.BENCHMARK_INSTANCE_IP }}":/tmp/litentry-parachain/runtime/$c/src/weights/*.rs runtime/$c/src/weights/
+            scp -o StrictHostKeychecking=no -i ./benchmark-server-key.pem "${{ env.BENCHMARK_SSH_USER }}"@"${{ steps.start_instance.outputs.remote_ip }}":/tmp/litentry-parachain/runtime/$c/src/weights/*.rs runtime/$c/src/weights/
           done
           echo "======================"
           git status
 
       - name: Stop remote instance
         if: always()
         run: |
-          aws ec2 stop-instances --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }}
+          aws ec2 stop-instances --instance-ids ${{ env.INSTANCE_ID }}
           sleep 5
-          ret=`aws ec2 describe-instance-status --region ap-southeast-1 --instance-ids ${{ env.INSTANCE_ID }} | jq '.InstanceStatuses[0].InstanceState.Name'`
+          ret=`aws ec2 describe-instance-status --instance-ids ${{ env.INSTANCE_ID }} | jq '.InstanceStatuses[0].InstanceState.Name'`
           echo "Remote instance running state: $ret"
 
       - name: Create auto PR

diff --git a/bitacross-worker/docker/docker-compose.yml b/bitacross-worker/docker/docker-compose.yml
@@ -134,6 +134,7 @@ services:
       - KARAT_DAO_API_URL=http://localhost:19527/karat_dao/
       - MAGIC_CRAFT_API_URL=http://localhost:19527/magic_craft/
       - MAGIC_CRAFT_API_KEY=
+      - DAREN_MARKET_API_URL=http://localhost:19527/daren_market/
       - MORALIS_API_KEY=
       - NODEREAL_API_KEY=NODEREAL_API_KEY
       - NODEREAL_API_URL=http://localhost:19527

diff --git a/bitacross-worker/docker/multiworker-docker-compose.yml b/bitacross-worker/docker/multiworker-docker-compose.yml
@@ -135,6 +135,7 @@ services:
       - KARAT_DAO_API_URL=http://localhost:19527/karat_dao/
       - MAGIC_CRAFT_API_URL=http://localhost:19527/magic_craft/
       - MAGIC_CRAFT_API_KEY=
+      - DAREN_MARKET_API_URL=http://localhost:19527/daren_market/
       - MORALIS_API_KEY=
       - NODEREAL_API_KEY=NODEREAL_API_KEY
       - NODEREAL_API_URL=http://localhost:19527
@@ -193,6 +194,7 @@ services:
       - KARAT_DAO_API_URL=http://localhost:19527/karat_dao/
       - MAGIC_CRAFT_API_URL=http://localhost:19527/magic_craft/
       - MAGIC_CRAFT_API_KEY=
+      - DAREN_MARKET_API_URL=http://localhost:19527/daren_market/
       - MORALIS_API_KEY=
       - NODEREAL_API_KEY=NODEREAL_API_KEY
       - NODEREAL_API_URL=http://localhost:19527

diff --git a/local-setup/.env.dev b/local-setup/.env.dev
@@ -61,4 +61,5 @@ MORALIS_API_URL=http://localhost:19527/moralis/
 MORALIS_SOLANA_API_URL=http://localhost:19527/moralis_solana/
 KARAT_DAO_API_URL=http://localhost:19527/karat_dao/
 MAGIC_CRAFT_API_URL=http://localhost:19527/magic_craft/
+DAREN_MARKET_API_URL=http://localhost:19527/daren_market/
 BLOCKCHAIN_INFO_API_URL=http://localhost:19527/blockchain_info/
diff --git a/primitives/core/src/assertion/platform_user.rs b/primitives/core/src/assertion/platform_user.rs
@@ -23,15 +23,17 @@ use crate::assertion::network::{all_evm_web3networks, Web3Network};
 #[derive(Encode, Decode, Clone, Debug, PartialEq, Eq, MaxEncodedLen, TypeInfo)]
 pub enum PlatformUserType {
 	#[codec(index = 0)]
-	KaratDaoUser,
+	KaratDao,
 	#[codec(index = 1)]
-	MagicCraftStakingUser,
+	MagicCraftStaking,
+	#[codec(index = 2)]
+	DarenMarket,
 }
 
 impl PlatformUserType {
 	pub fn get_supported_networks(&self) -> Vec<Web3Network> {
 		match self {
-			Self::KaratDaoUser | Self::MagicCraftStakingUser => all_evm_web3networks(),
+			Self::KaratDao | Self::MagicCraftStaking | Self::DarenMarket => all_evm_web3networks(),
 		}
 	}
 }
diff --git a/scripts/pre-commit.sh b/scripts/pre-commit.sh
@@ -36,6 +36,10 @@ function clean_up() {
     cargo clean
     cd "$root_dir/tee-worker/enclave-runtime"
     cargo clean
+    cd "$root_dir/bitacross-worker"
+    cargo clean
+    cd "$root_dir/bitacross-worker/enclave-runtime"
+    cargo clean
 }
 
 root_dir=$(git rev-parse --show-toplevel)

diff --git a/tee-worker/cli/src/trusted_base_cli/commands/litentry/request_vc.rs b/tee-worker/cli/src/trusted_base_cli/commands/litentry/request_vc.rs
@@ -284,8 +284,9 @@ pub enum TokenHoldingAmountCommand {
 
 #[derive(Subcommand, Debug)]
 pub enum PlatformUserCommand {
-	KaratDaoUser,
-	MagicCraftStakingUser,
+	KaratDao,
+	MagicCraftStaking,
+	DarenMarket,
 }
 
 #[derive(Subcommand, Debug)]
@@ -652,9 +653,10 @@ impl Command {
 				TokenHoldingAmountCommand::Tuna => TokenHoldingAmount(Web3TokenType::Tuna),
 			}),
 			Command::PlatformUser(arg) => Ok(match arg {
-				PlatformUserCommand::KaratDaoUser => PlatformUser(PlatformUserType::KaratDaoUser),
-				PlatformUserCommand::MagicCraftStakingUser =>
-					PlatformUser(PlatformUserType::MagicCraftStakingUser),
+				PlatformUserCommand::KaratDao => PlatformUser(PlatformUserType::KaratDao),
+				PlatformUserCommand::MagicCraftStaking =>
+					PlatformUser(PlatformUserType::MagicCraftStaking),
+				PlatformUserCommand::DarenMarket => PlatformUser(PlatformUserType::DarenMarket),
 			}),
 			Command::NftHolder(arg) => Ok(match arg {
 				NftHolderCommand::WeirdoGhostGang => NftHolder(Web3NftType::WeirdoGhostGang),

diff --git a/tee-worker/client-api/parachain-api/prepare-build/interfaces/vc/definitions.ts b/tee-worker/client-api/parachain-api/prepare-build/interfaces/vc/definitions.ts
@@ -242,7 +242,7 @@ export default {
         },
         // PlatformUserType
         PlatformUserType: {
-            _enum: ["KaratDaoUser", "MagicCraftStakingUser"],
+            _enum: ["KaratDao", "MagicCraftStaking", "DarenMarket"],
         },
         // Web3NftType
         Web3NftType: {

diff --git a/tee-worker/docker/docker-compose.yml b/tee-worker/docker/docker-compose.yml
@@ -138,6 +138,7 @@ services:
       - KARAT_DAO_API_URL=http://localhost:19527/karat_dao/
       - MAGIC_CRAFT_API_URL=http://localhost:19527/magic_craft/
       - MAGIC_CRAFT_API_KEY=
+      - DAREN_MARKET_API_URL=http://localhost:19527/daren_market/
       - MORALIS_API_KEY=
       - NODEREAL_API_KEY=NODEREAL_API_KEY
       - NODEREAL_API_URL=http://localhost:19527

diff --git a/tee-worker/docker/multiworker-docker-compose.yml b/tee-worker/docker/multiworker-docker-compose.yml
@@ -139,6 +139,7 @@ services:
       - KARAT_DAO_API_URL=http://localhost:19527/karat_dao/
       - MAGIC_CRAFT_API_URL=http://localhost:19527/magic_craft/
       - MAGIC_CRAFT_API_KEY=
+      - DAREN_MARKET_API_URL=http://localhost:19527/daren_market/
       - MORALIS_API_KEY=
       - NODEREAL_API_KEY=NODEREAL_API_KEY
       - NODEREAL_API_URL=http://localhost:19527
@@ -203,6 +204,7 @@ services:
       - KARAT_DAO_API_URL=http://localhost:19527/karat_dao/
       - MAGIC_CRAFT_API_URL=http://localhost:19527/magic_craft/
       - MAGIC_CRAFT_API_KEY=
+      - DAREN_MARKET_API_URL=http://localhost:19527/daren_market/
       - MORALIS_API_KEY=
       - NODEREAL_API_KEY=NODEREAL_API_KEY
       - NODEREAL_API_URL=http://localhost:19527
@@ -267,6 +269,7 @@ services:
       - KARAT_DAO_API_URL=http://localhost:19527/karat_dao/
       - MAGIC_CRAFT_API_URL=http://localhost:19527/magic_craft/
       - MAGIC_CRAFT_API_KEY=
+      - DAREN_MARKET_API_URL=http://localhost:19527/daren_market/
       - MORALIS_API_KEY=
       - NODEREAL_API_KEY=NODEREAL_API_KEY
       - NODEREAL_API_URL=http://localhost:19527