Skip to content

Commit 17b1e85

Browse files
NathanBSCallformless
NathanBSC
authored and
allformless
committed
support QA deploy
1 parent e2a0f09 commit 17b1e85

File tree

9 files changed

+391
-19
lines changed

9 files changed

+391
-19
lines changed

‎.env‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ INIT_HOLDER="0x04d63aBCd2b9b1baa327f2Dda0f873F197ccd186"
55
# INIT_HOLDER_PRV="59ba8068eb256d520179e903f43dacf6d8d57d72bd306e1bd603fdb8c8da10e8"
66
RPC_URL="http://127.0.0.1:8545"
77
GENESIS_COMMIT="234e3685ec309624f0fbef41043ae117caebc853" # lorentz commit
8-
PASSED_FORK_DELAY=40
8+
PASSED_FORK_DELAY=600
99
LAST_FORK_MORE_DELAY=10
1010
FullImmutabilityThreshold=2048
1111
MinBlocksForBlobRequests=576
1212
DefaultExtraReserveForBlobRequests=32
1313
BreatheBlockInterval=1200
1414
useLatestBscClient=false
15-
EnableSentryNode=false
16-
EnableFullNode=false
17-
RegisterNodeID=false
18-
EnableEVNWhitelist=false
15+
EnableSentryNode=true
16+
EnableFullNode=true
17+
RegisterNodeID=true
18+
EnableEVNWhitelist=true

‎bsc_cluster.sh‎

Lines changed: 133 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,15 @@ basedir=$(
99
)
1010
workspace=${basedir}
1111
source ${workspace}/.env
12-
size=$((BSC_CLUSTER_SIZE))
12+
source ${workspace}/qa-env-resource/machines_meta.sh # including machine ips and ids, don't upload!!!
13+
validator_ips=(${validator_ips_comma//,/ })
14+
size=${#validator_ips[@]}
1315
stateScheme="hash"
1416
dbEngine="leveldb"
1517
gcmode="full"
1618
sleepBeforeStart=15
1719
sleepAfterStart=10
20+
copyDir="bsc-qa"
1821

1922
# stop geth client
2023
function exit_previous() {
@@ -57,7 +60,7 @@ function reset_genesis() {
5760
mv genesis-template.json.bk genesis-template.json
5861
mv scripts/init_holders.template.bk scripts/init_holders.template
5962

60-
poetry install --no-root
63+
# poetry install --no-root
6164
npm install
6265
rm -rf lib/forge-std
6366
forge install --no-git --no-commit foundry-rs/forge-std@v1.7.3
@@ -83,28 +86,49 @@ function prepare_config() {
8386
mkdir -p ${targetDir} && cd ${targetDir}
8487
cp ${workspace}/keys/password.txt ./
8588
cp ${workspace}/.local/hardforkTime.txt ./
89+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
90+
sed -i -e "s/{{validatorAddr}}/${cons_addr}/g" chaind.sh && rm -f chaind.sh.bak
8691
bbcfee_addrs=${fee_addr}
8792
powers="0x000001d1a94a2000" #2000000000000
8893
mv ${workspace}/.local/bls${i}/bls ./ && rm -rf ${workspace}/.local/bls${i}
8994
vote_addr=0x$(cat ./bls/keystore/*json | jq .pubkey | sed 's/"//g')
9095
echo "${cons_addr},${bbcfee_addrs},${fee_addr},${powers},${vote_addr}" >> ${workspace}/genesis/validators.conf
9196
if [ ${EnableSentryNode} = true ]; then
92-
mkdir -p ${workspace}/.local/sentry${i}
97+
targetDir=${workspace}/.local/sentry${i}
98+
mkdir -p ${targetDir} && cd ${targetDir}
99+
cp ${workspace}/.local/hardforkTime.txt ./
100+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
101+
sed -i -e '/--mine/d' chaind.sh
102+
sed -i -e 's/workdir="validator"/workdir="sentry"/g' chaind.sh
103+
sed -i -e 's/bin="bsc"/bin="sentry"/g' chaind.sh
104+
sed -i -e "s/portInc=0/portInc=2/g" chaind.sh
105+
rm -f chaind.sh.bak
106+
sed -i -e 's/workdir="validator"/workdir="sentry"/g' init.sh
107+
sed -i -e 's/bin="bsc"/bin="sentry"/g' init.sh
108+
rm -f init.sh.bak
109+
mv bsc.service sentry.service
110+
sed -i -e 's/validator/sentry/g' sentry.service
111+
sed -i -e 's/bsc/sentry/g' sentry.service
112+
rm -f sentry.service.bak
93113
fi
94114
done
95115
if [ ${EnableFullNode} = true ]; then
96-
mkdir -p ${workspace}/.local/fullnode0
116+
targetDir=${workspace}/.local/fullnode0
117+
mkdir -p ${targetDir} && cd ${targetDir}
118+
cp ${workspace}/.local/hardforkTime.txt ./
119+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
120+
sed -i -e '/--mine/d' chaind.sh && rm -f chaind.sh.bak
97121
fi
98122
rm -f ${workspace}/.local/hardforkTime.txt
99123

100124
cd ${workspace}/genesis/
101125
git checkout HEAD contracts
102126
sed -i -e 's/alreadyInit = true;/turnLength = 16;alreadyInit = true;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
103127
sed -i -e 's/public onlyCoinbase onlyZeroGasPrice {/public onlyCoinbase onlyZeroGasPrice {if (block.number < 300) return;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
104-
105-
poetry run python -m scripts.generate generate-validators
106-
poetry run python -m scripts.generate generate-init-holders "${initHolders}"
107-
poetry run python -m scripts.generate dev \
128+
129+
python3 -m scripts.generate generate-validators
130+
python3 -m scripts.generate generate-init-holders "${initHolders}"
131+
python3 -m scripts.generate dev \
108132
--dev-chain-id "${CHAIN_ID}" \
109133
--init-burn-ratio "1000" \
110134
--init-felony-slash-scope "60" \
@@ -121,6 +145,7 @@ function prepare_config() {
121145
--governor-protector "${INIT_HOLDER}" \
122146
--init-minimal-delay "1 minutes" \
123147
--token-recover-portal-protector "${INIT_HOLDER}"
148+
#cp genesis-dev.json genesis.json
124149
}
125150

126151
function initNetwork() {
@@ -141,10 +166,10 @@ function initNetwork() {
141166

142167
init_extra_args=""
143168
if [ ${EnableSentryNode} = true ]; then
144-
init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ports 30411"
169+
init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ips ${sentry_ips_comma}"
145170
fi
146171
if [ ${EnableFullNode} = true ]; then
147-
init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ports 30511"
172+
init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ips ${fullnode_ips_comma}"
148173
fi
149174
if [ "${RegisterNodeID}" = true ]; then
150175
if [ "${EnableSentryNode}" = true ]; then
@@ -160,7 +185,7 @@ function initNetwork() {
160185
init_extra_args="${init_extra_args} --init.evn-validator-whitelist"
161186
fi
162187
fi
163-
${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --config ${workspace}/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
188+
${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --init.ips "${validator_ips_comma}" --config ${workspace}/qa-env-resource/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
164189
rm -f ${workspace}/*bsc.log*
165190
for ((i = 0; i < size; i++)); do
166191
sed -i -e '/"<nil>"/d' ${workspace}/.local/node${i}/config.toml
@@ -176,14 +201,21 @@ function initNetwork() {
176201
rm -f ${workspace}/.local/node${i}/*bsc.log*
177202

178203
if [ ${EnableSentryNode} = true ]; then
204+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/node${i}/config.toml
205+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/node${i}/config.toml
206+
179207
sed -i -e '/"<nil>"/d' ${workspace}/.local/sentry${i}/config.toml
208+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/sentry${i}/config.toml
209+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/sentry${i}/config.toml
180210
initLog=${workspace}/.local/sentry${i}/init.log
181211
${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1
182212
rm -f ${workspace}/.local/sentry${i}/*bsc.log*
183213
fi
184214
done
185215
if [ ${EnableFullNode} = true ]; then
186216
sed -i -e '/"<nil>"/d' ${workspace}/.local/fullnode0/config.toml
217+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/fullnode0/config.toml
218+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/fullnode0/config.toml
187219
sed -i -e 's/EnableEVNFeatures = true/EnableEVNFeatures = false/g' ${workspace}/.local/fullnode0/config.toml
188220
initLog=${workspace}/.local/fullnode0/init.log
189221
${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1
@@ -267,13 +299,76 @@ function native_start() {
267299

268300
function register_stakehub(){
269301
# wait feynman enable
270-
sleep 45
302+
sleep 100
271303
for ((i = 0; i < size; i++));do
272304
${workspace}/create-validator/create-validator --consensus-key-dir ${workspace}/keys/validator${i} --vote-key-dir ${workspace}/keys/bls${i} \
273305
--password-path ${workspace}/keys/password.txt --amount 20001 --validator-desc Val${i} --rpc-url ${RPC_URL}
274306
done
275307
}
276308

309+
function remote_reset_config() {
310+
rm -rf /mnt/efs/${copyDir}/clusterNetwork
311+
cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
312+
ips=(${validator_ips_comma//,/ })
313+
for ((i=0;i<${#ips[@]};i++));do
314+
dst_id=${ips2ids[${ips[i]}]}
315+
if [ ${EnableSentryNode} = true ]; then
316+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/config.toml /server/sentry/"
317+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/chaind.sh /server/sentry/"
318+
fi
319+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/node${i}/config.toml /server/validator/"
320+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/node${i}/chaind.sh /server/validator/"
321+
done
322+
if [ ${EnableFullNode} = true ]; then
323+
fullnode_ips=(${fullnode_ips_comma//,/ })
324+
dst_id=${ips2ids[${fullnode_ips[0]}]}
325+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/fullnode0/config.toml /server/validator/"
326+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/fullnode0/chaind.sh /server/validator/"
327+
fi
328+
}
329+
330+
function remote_start() {
331+
rm -rf /mnt/efs/${copyDir}/clusterNetwork
332+
cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
333+
for dst_id in ${ips2ids[@]}; do
334+
if [ ${EnableSentryNode} = true ]; then
335+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo service sentry stop"
336+
fi
337+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo service bsc stop"
338+
done
339+
sleep 100
340+
cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
341+
ips=(${validator_ips_comma//,/ })
342+
for ((i=0;i<${#ips[@]};i++));do
343+
dst_id=${ips2ids[${ips[i]}]}
344+
if [ ${EnableSentryNode} = true ]; then
345+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/init.sh"
346+
fi
347+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/node${i}/init.sh"
348+
done
349+
if [ ${EnableFullNode} = true ]; then
350+
fullnode_ips=(${fullnode_ips_comma//,/ })
351+
dst_id=${ips2ids[${fullnode_ips[0]}]}
352+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/fullnode0/init.sh"
353+
fi
354+
}
355+
356+
function remote_upgrade() {
357+
cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
358+
if [ ${EnableSentryNode} = true ]; then
359+
cp ${workspace}/qa-env-resource/upgrade-single-sentry.sh /mnt/efs/${copyDir}/clusterNetwork/
360+
fi
361+
cp ${workspace}/qa-env-resource/upgrade-single-validator.sh /mnt/efs/${copyDir}/clusterNetwork/
362+
for dst_id in ${ips2ids[@]}; do
363+
if [ ${EnableSentryNode} = true ]; then
364+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
365+
--parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-sentry.sh /tmp/ && sudo bash -x /tmp/upgrade-single-sentry.sh"
366+
fi
367+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
368+
--parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-validator.sh /tmp/ && sudo bash -x /tmp/upgrade-single-validator.sh"
369+
done
370+
}
371+
277372
CMD=$1
278373
ValidatorIdx=$2
279374
case ${CMD} in
@@ -284,7 +379,7 @@ reset)
284379
reset_genesis
285380
prepare_config
286381
initNetwork
287-
native_start
382+
native_start
288383
register_stakehub
289384
;;
290385
stop)
@@ -297,7 +392,31 @@ restart)
297392
exit_previous $ValidatorIdx
298393
native_start $ValidatorIdx
299394
;;
395+
remote_reset)
396+
create_validator
397+
reset_genesis
398+
prepare_config
399+
initNetwork
400+
remote_start
401+
# to prevent stuck
402+
sleep 10
403+
remote_upgrade
404+
register_stakehub
405+
;;
406+
remote_reset_config)
407+
create_validator
408+
reset_genesis
409+
prepare_config
410+
initNetwork
411+
remote_reset_config
412+
;;
413+
remote_upgrade)
414+
remote_upgrade
415+
;;
416+
register_stakehub)
417+
register_stakehub
418+
;;
300419
*)
301-
echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]"
420+
echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]| remote_reset | remote_upgrade | register_stakehub"
302421
;;
303422
esac

‎qa-env-resource/bsc.service‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[Unit]
2+
Description=bsc
3+
After=network.target
4+
5+
[Service]
6+
Type=simple
7+
User=root
8+
Group=root
9+
ExecStart=/server/validator/chaind.sh -start
10+
ExecReload=/server/validator/chaind.sh -restart
11+
ExecStop=/server/validator/chaind.sh -stop
12+
PrivateTmp=true
13+
LimitNOFILE=10000
14+
StartLimitInterval=0
15+
TimeoutStopSec=120
16+
17+
[Install]
18+
WantedBy=multi-user.target

‎qa-env-resource/chaind.sh‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
export GOGC=200
6+
# default values
7+
workdir="validator"
8+
bin="bsc"
9+
portInc=0
10+
FullImmutabilityThreshold=90000
11+
MinBlocksForBlobRequests=524288
12+
DefaultExtraReserveForBlobRequests=28800
13+
BreatheBlockInterval=600
14+
LAST_FORK_MORE_DELAY=200
15+
16+
function startChaind() {
17+
workspace=/server/${workdir}
18+
19+
PassedForkTime=`cat ${workspace}/hardforkTime.txt|grep passedHardforkTime|awk -F" " '{print $NF}'`
20+
LastHardforkTime=$(expr ${PassedForkTime} + ${LAST_FORK_MORE_DELAY})
21+
initLog=${workspace}/init.log
22+
rialtoHash=`cat ${initLog}|grep "database=chaindata"|awk -F"=" '{print $NF}'|awk -F'"' '{print $1}'`
23+
24+
ip=`ifconfig eth0|grep inet|grep -v inet6 |awk '{ print $2 }'`
25+
sed -i -e "s?FileRoot = \"\"?FileRoot = \"/mnt/efs/${workdir}/${ip}/\"?g" /server/${workdir}/config.toml
26+
mkdir -p /mnt/efs/${workdir}/${ip}
27+
HTTPPort=$((8545 + ${portInc}))
28+
WSPort=${HTTPPort}
29+
MetricsPort=$((6060 + ${portInc}))
30+
PProfPort=$((${MetricsPort} + 1))
31+
${workspace}/${bin} --config ${workspace}/config.toml \
32+
--mine --vote --unlock {{validatorAddr}} --miner.etherbase {{validatorAddr}} --password ${workspace}/password.txt --blspassword ${workspace}/password.txt \
33+
--datadir ${workspace} \
34+
--rpc.allow-unprotected-txs --allow-insecure-unlock \
35+
--ws --ws.addr ${ip} --ws.port ${WSPort} --http.addr 0.0.0.0 --http.port ${HTTPPort} --http.corsdomain "*" \
36+
--metrics --metrics.addr 0.0.0.0 --metrics.port ${MetricsPort} \
37+
--pprof --pprof.port ${PProfPort} \
38+
--syncmode full --monitor.maliciousvote \
39+
--cache 10480 \
40+
--rialtohash ${rialtoHash} --override.passedforktime ${PassedForkTime} --override.lorentz ${PassedForkTime} --override.maxwell ${LastHardforkTime} \
41+
--override.immutabilitythreshold ${FullImmutabilityThreshold} --override.breatheblockinterval ${BreatheBlockInterval} \
42+
--override.minforblobrequest ${MinBlocksForBlobRequests} --override.defaultextrareserve ${DefaultExtraReserveForBlobRequests} \
43+
>> /mnt/efs/${workdir}/${ip}/bscnode.log 2>&1
44+
}
45+
46+
function stopChaind() {
47+
pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
48+
if [ -n "$pid" ]; then
49+
kill -TERM $pid
50+
for((i=1;i<=40;i++));
51+
do
52+
pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
53+
if [ -z "$pid" ]; then
54+
break
55+
fi
56+
sleep 10
57+
done
58+
fi
59+
}
60+
61+
CMD=$1
62+
63+
case $CMD in
64+
-start)
65+
echo "start"
66+
startChaind
67+
;;
68+
-stop)
69+
echo "stop"
70+
stopChaind
71+
;;
72+
-restart)
73+
stopChaind
74+
sleep 3
75+
startChaind
76+
;;
77+
*)
78+
echo "Usage: chaind.sh -start | -stop | -restart .Or use systemctl start | stop | restart ${bin}.service "
79+
;;
80+
esac

0 commit comments

Comments
 (0)