support QA deploy

Author: NathanBSC

.env

@@ -5,7 +5,7 @@ KEYPASS="0123456789"
 INIT_HOLDER="0x04d63aBCd2b9b1baa327f2Dda0f873F197ccd186"
 INIT_HOLDER_PRV="59ba8068eb256d520179e903f43dacf6d8d57d72bd306e1bd603fdb8c8da10e8"
 RPC_URL="http://127.0.0.1:8545"
-HARD_FORK_DELAY=40
+HARD_FORK_DELAY=200
 FullImmutabilityThreshold=512
 MinBlocksForBlobRequests=576
 DefaultExtraReserveForBlobRequests=32

.gitignore

@@ -6,4 +6,6 @@
 
 .local/
 
-.idea/
+.idea/
+
+/qa-env-resource/machines_meta.sh

config.toml

@@ -19,7 +19,7 @@ GasPrice = 10000000000
 Recommit = 10000000000
 
 [Eth.Miner.Mev]
-Enabled = false
+Enabled = true
 GreedyMergeTx = true
 BuilderFeeCeil= "0"
 ValidatorCommission = 1000

qa-env-resource/bsc.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=bsc
+After=network.target
+
+[Service]
+Type=simple
+User=root
+Group=root
+ExecStart=/server/validator/chaind.sh -start
+ExecReload=/server/validator/chaind.sh -restart
+ExecStop=/server/validator/chaind.sh -stop
+PrivateTmp=true
+Restart=always
+LimitNOFILE=10000
+RestartSec=5
+StartLimitInterval=0
+
+[Install]
+WantedBy=multi-user.target

qa-env-resource/chaind.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+export GOGC=200
+# default values
+FullImmutabilityThreshold=90000
+MinBlocksForBlobRequests=524288
+DefaultExtraReserveForBlobRequests=28800
+BreatheBlockInterval=600
+
+function startChaind() {
+    workspace=/server/validator
+
+    FeynmanHardforkTime=`cat ${workspace}/hardforkTime.txt|grep hardforkTime|awk -F" " '{print $NF}'`
+    CancunHardforkTime=`expr ${FeynmanHardforkTime} + 10`
+    initLog=${workspace}/init.log
+    rialtoHash=`cat ${initLog}|grep "database=lightchaindata"|awk -F"=" '{print $NF}'|awk -F'"' '{print $1}'`
+
+    ip=`ifconfig eth0|grep inet|grep -v inet6 |awk '{ print $2 }'`
+    sed -i -e "s?FileRoot = \"\"?FileRoot = \"/mnt/efs/validator/${ip}/\"?g" /server/validator/config.toml
+    mkdir -p /mnt/efs/validator/${ip}
+    ${workspace}/bsc --config ${workspace}/config.toml \
+        --datadir ${workspace} \
+        --password ${workspace}/password.txt \
+        --blspassword ${workspace}/password.txt \
+        --unlock {{validatorAddr}} --miner.etherbase {{validatorAddr}} --rpc.allow-unprotected-txs --allow-insecure-unlock \
+        --ws --ws.port 8545 --ws.addr ${ip} --http.addr 0.0.0.0 --http.corsdomain "*" \
+        --metrics --metrics.addr 0.0.0.0 \
+        --pprof --pprof.port 6061 \
+        --syncmode snap --mine --vote --monitor.maliciousvote \
+        --cache 10480 --light.serve 50 \
+        --rialtohash ${rialtoHash} --override.cancun ${CancunHardforkTime} --override.haber ${CancunHardforkTime} \
+        --override.immutabilitythreshold ${FullImmutabilityThreshold} --override.breatheblockinterval ${BreatheBlockInterval} \
+        --override.minforblobrequest ${MinBlocksForBlobRequests} --override.defaultextrareserve ${DefaultExtraReserveForBlobRequests} \
+        >> /mnt/efs/validator/${ip}/bscnode.log 2>&1
+}
+
+function stopChaind() {
+    pid=`ps -ef | grep /server/validator/bsc | grep -v grep | awk '{print $2}'`
+    if [ -n "$pid" ]; then
+        for((i=1;i<=4;i++));
+        do
+            kill $pid
+            sleep 5
+            pid=`ps -ef | grep /server/validator/bsc | grep -v grep | awk '{print $2}'`
+            if [ -z "$pid" ]; then
+                break
+            elif [ $i -eq 4 ]; then
+                kill -9 $kid
+            fi
+        done
+    fi
+}
+
+CMD=$1
+
+case $CMD in
+-start)
+    echo "start"
+    startChaind
+    ;;
+-stop)
+    echo "stop"
+    stopChaind
+    ;;
+-restart)
+    stopChaind
+    sleep 3
+    startChaind
+    ;;
+*)
+    echo "Usage: chaind.sh -start | -stop | -restart .Or use systemctl start | stop | restart bsc.service "
+    ;;
+esac

qa-env-resource/config.toml

@@ -0,0 +1,65 @@
+[Eth]
+NetworkId = 714
+SyncMode = "full"
+NoPruning = false
+NoPrefetch = false
+LightPeers = 100
+TrieTimeout = 100000000000
+DatabaseCache = 512
+DatabaseFreezer = ""
+TriesInMemory = 128
+TrieCleanCache = 256
+TrieDirtyCache = 256
+EnablePreimageRecording = false
+
+[Eth.Miner]
+GasFloor = 500000000
+GasCeil = 500000000
+GasPrice = 1000000000
+Recommit = 10000000000
+
+[Eth.TxPool]
+Locals = []
+NoLocals = true
+Journal = "transactions.rlp"
+Rejournal = 3600000000000
+PriceLimit = 1000000000
+PriceBump = 10
+AccountSlots = 512
+GlobalSlots = 10000
+AccountQueue = 256
+GlobalQueue = 5000
+Lifetime = 10800000000000
+
+[Eth.GPO]
+Blocks = 20
+Percentile = 60
+OracleThreshold = 1000
+
+[Node]
+IPCPath = "geth.ipc"
+HTTPHost = "localhost"
+NoUSB = true
+InsecureUnlockAllowed = false
+HTTPPort = 8545
+HTTPVirtualHosts = ["localhost"]
+HTTPModules = ["eth", "net", "web3", "txpool", "parlia", "miner"]
+WSPort = 8546
+WSModules = ["net", "web3", "eth"]
+
+[Node.P2P]
+MaxPeers = 30
+NoDiscovery = false
+ListenAddr = ":30311"
+EnableMsgEvents = false
+
+[Node.HTTPTimeouts]
+ReadTimeout = 30000000000
+WriteTimeout = 30000000000
+IdleTimeout = 120000000000
+
+[Node.LogConfig]
+FilePath = "bsc.log"
+MaxBytesSize = 10485760
+Level = "info"
+FileRoot = ""

qa-env-resource/init.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+workspace=$(cd `dirname $0`; pwd)
+
+cp ${workspace}/bsc.service /usr/lib/systemd/system/
+chmod +x ${workspace}/start.sh ${workspace}/chaind.sh ${workspace}/bsc
+
+service bsc stop
+rm -rf /server/validator
+mv  ${workspace} /server/validator
+
+/server/validator/start.sh

qa-env-resource/start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+systemctl daemon-reload
+chkconfig bsc on
+service bsc restart

qa-env-resource/upgrade-single.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+service bsc stop
+cp /tmp/bsc /server/validator/bsc
+
+#cd  /server/validator/geth/
+#rm -rf  chaindata  les.server  nodes  triecache
+
+#sed -i -e 's/sleep 5/sleep 40/' /server/validator/chaind.sh
+
+#/server/validator/bsc init --datadir /server/validator/ /server/validator/genesis.json
+
+#sed -i -e 's/HTTPModules = \[/HTTPModules = \["debug",/g' /server/validator/config.toml
+
+service bsc start

start_cluster.sh

@@ -9,13 +9,14 @@ basedir=$(
 )
 workspace=${basedir}
 source ${workspace}/.env
-size=$((BSC_CLUSTER_SIZE))
-stateScheme="path"
-dbEngine="pebble"
+source ${workspace}/qa-env-resource/machines_meta.sh # including machine ips and ids, don't upload!!!
+size=${#ips2ids[@]}
+stateScheme="hash"
+dbEngine="leveldb"
 gcmode="full"
 epoch=200
 blockInterval=3
-needRegister=false
+needRegister=true
 sleepBeforeStart=10
 
 # stop geth client
@@ -48,14 +49,13 @@ function reset_genesis() {
         mv genesis-template.json.bk genesis-template.json
     fi
 
-    poetry install --no-root
+    # poetry install --no-root
     npm install
     rm -rf lib/forge-std
     forge install --no-git --no-commit foundry-rs/forge-std@v1.7.3
     cd lib/forge-std/lib
     rm -rf ds-test
     git clone https://github.com/dapphub/ds-test
-
 }
 
 function prepare_config() {
@@ -72,6 +72,10 @@ function prepare_config() {
         done
 
         mkdir -p ${workspace}/.local/bsc/node${i}
+        cp ${workspace}/.local/bsc/password.txt ${workspace}/.local/bsc/node${i}/
+        cp ${workspace}/qa-env-resource/* ${workspace}/.local/bsc/node${i}/
+        sed -i -e "s/{{validatorAddr}}/${cons_addr}/g" ${workspace}/.local/bsc/node${i}/chaind.sh
+        cp ${workspace}/.local/bsc/hardforkTime.txt ${workspace}/.local/bsc/node${i}/hardforkTime.txt
         bbcfee_addrs=${fee_addr}
         powers="0x000001d1a94a2000" #2000000000000
         mv ${workspace}/.local/bsc/bls${i}/bls ${workspace}/.local/bsc/node${i}/ && rm -rf ${workspace}/.local/bsc/bls${i}
@@ -88,10 +92,10 @@ function prepare_config() {
         sed -i -e '/registeredContractChannelMap\[VALIDATOR_CONTRACT_ADDR\]\[STAKING_CHANNELID\]/d' ${workspace}/genesis/contracts/CrossChain.sol
         sed -i -e  's/public onlyCoinbase onlyZeroGasPrice {/public onlyCoinbase onlyZeroGasPrice {if (block.number < 30) return;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
     fi
-    poetry run python -m scripts.generate generate-validators
-    poetry run python -m scripts.generate generate-init-holders "${initHolders}"
-    poetry run python -m scripts.generate dev --dev-chain-id ${BSC_CHAIN_ID} --whitelist-1 "${INIT_HOLDER}" \
-      --epoch ${epoch} --misdemeanor-threshold "5" --felony-threshold "10" \
+    python3 -m scripts.generate generate-validators
+    python3 -m scripts.generate generate-init-holders "${initHolders}"
+    python3 -m scripts.generate dev --dev-chain-id ${BSC_CHAIN_ID} --whitelist-1 "${INIT_HOLDER}" \
+      --epoch ${epoch} \
       --init-felony-slash-scope "60" \
       --breathe-block-interval "1 minutes" \
       --block-interval ${blockInterval} \
@@ -110,16 +114,21 @@ function prepare_config() {
 
 function initNetwork() {
     cd ${workspace}
-    ${workspace}/bin/geth init-network --init.dir ${workspace}/.local/bsc --init.size=${size} --config ${workspace}/config.toml ${workspace}/genesis/genesis.json
+    ${workspace}/bin/geth init-network --init.dir ${workspace}/.local/bsc --init.size=${size} --init.ips "${validator_ips_comma}" --config ${workspace}/qa-env-resource/config.toml ${workspace}/genesis/genesis.json
     rm -rf ${workspace}/*bsc.log*
     for ((i = 0; i < size; i++)); do
         sed -i -e '/"<nil>"/d' ${workspace}/.local/bsc/node${i}/config.toml
         cp -R ${workspace}/.local/bsc/validator${i}/keystore ${workspace}/.local/bsc/node${i}
 
-        cp ${workspace}/bin/geth ${workspace}/.local/bsc/node${i}/geth${i}
+        #cp ${workspace}/bin/geth ${workspace}/.local/bsc/node${i}/geth${i}
+        cp ${workspace}/bin/geth ${workspace}/.local/bsc/node${i}/bsc
         # init genesis
         initLog=${workspace}/.local/bsc/node${i}/init.log
-        ${workspace}/bin/geth --datadir ${workspace}/.local/bsc/node${i} init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json  > "${initLog}" 2>&1
+        if  [ `expr $i \* 2` -ge ${size} ] ; then
+            ${workspace}/bin/geth --datadir ${workspace}/.local/bsc/node${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json  > "${initLog}" 2>&1
+        else
+            ${workspace}/bin/geth --datadir ${workspace}/.local/bsc/node${i} init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json  > "${initLog}" 2>&1
+        fi
     done
 }
 
@@ -175,6 +184,25 @@ function register_stakehub(){
     fi
 }
 
+function remote_start() {
+    rm -rf /mnt/efs/bsc-qa/clusterNetwork
+    cp -r ${workspace}/.local/bsc /mnt/efs/bsc-qa/clusterNetwork
+    ips=(${validator_ips_comma//,/ })
+    for ((i=0;i<${#ips[@]};i++));do
+        dst_id=${ips2ids[${ips[i]}]}
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript"   --parameters commands="sudo bash -x /mnt/efs/bsc-qa/clusterNetwork/node${i}/init.sh"
+    done
+}
+
+function remote_upgrade() {
+    cp ${workspace}/bin/geth /mnt/efs/bsc-qa/clusterNetwork/
+    cp ${workspace}/qa-env-resource/upgrade-single.sh /mnt/efs/bsc-qa/clusterNetwork/
+    for dst_id in ${ips2ids[@]}; do
+        aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
+            --parameters commands="sudo cp /mnt/efs/bsc-qa/clusterNetwork/geth /tmp/bsc && sudo cp /mnt/efs/bsc-qa/clusterNetwork/upgrade-single.sh /tmp/ && sudo bash -x /tmp/upgrade-single.sh"
+    done
+}
+
 ## docker relate begin
 function generate_static_peers() {
     tool=${workspace}/bin/bootnode
@@ -283,6 +311,17 @@ restart)
     exit_previous $ValidatorIdx
     native_start $ValidatorIdx
     ;;
+remote_reset)
+    create_validator
+    reset_genesis
+    prepare_config
+    initNetwork
+    remote_start
+    register_stakehub
+    ;;
+remote_upgrade)
+    remote_upgrade
+    ;;
 install_k8s)
     create_validator
     reset_genesis
@@ -295,6 +334,6 @@ uninstall_k8s)
     uninstall_k8s
     ;;
 *)
-    echo "Usage: start_cluster.sh | reset | stop | start | restart"
+    echo "Usage: start_cluster.sh | reset | stop | start | restart | remote_reset | remote_upgrade"
     ;;
 esac