Skip to content

Commit 92e42d1

Browse files
NathanBSCallformless
NathanBSC
authored and
allformless
committed
support QA deploy
1 parent bf77273 commit 92e42d1

File tree

9 files changed

+393
-20
lines changed

9 files changed

+393
-20
lines changed

‎.env‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ INIT_HOLDER="0x04d63aBCd2b9b1baa327f2Dda0f873F197ccd186"
55
# INIT_HOLDER_PRV="59ba8068eb256d520179e903f43dacf6d8d57d72bd306e1bd603fdb8c8da10e8"
66
RPC_URL="http://127.0.0.1:8545"
77
GENESIS_COMMIT="34618f607f8356cf147dde6a69fae150bd53d5bf" # fermi commit
8-
PASSED_FORK_DELAY=40
8+
PASSED_FORK_DELAY=600
99
LAST_FORK_MORE_DELAY=10
1010
FullImmutabilityThreshold=2048
1111
MinBlocksForBlobRequests=576
1212
DefaultExtraReserveForBlobRequests=32
1313
BreatheBlockInterval=1200
1414
useLatestBscClient=false
15-
EnableSentryNode=false
16-
EnableFullNode=false
17-
RegisterNodeID=false
18-
EnableEVNWhitelist=false
15+
EnableSentryNode=true
16+
EnableFullNode=true
17+
RegisterNodeID=true
18+
EnableEVNWhitelist=true

‎bsc_cluster.sh‎

Lines changed: 133 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,15 @@ basedir=$(
99
)
1010
workspace=${basedir}
1111
source ${workspace}/.env
12-
size=$((BSC_CLUSTER_SIZE))
12+
source ${workspace}/qa-env-resource/machines_meta.sh # including machine ips and ids, don't upload!!!
13+
validator_ips=(${validator_ips_comma//,/ })
14+
size=${#validator_ips[@]}
1315
stateScheme="hash"
1416
dbEngine="leveldb"
1517
gcmode="full"
1618
sleepBeforeStart=15
1719
sleepAfterStart=10
20+
copyDir="bsc-qa"
1821

1922
# stop geth client
2023
function exit_previous() {
@@ -57,7 +60,7 @@ function reset_genesis() {
5760
mv genesis-template.json.bk genesis-template.json
5861
mv scripts/init_holders.template.bk scripts/init_holders.template
5962

60-
poetry install --no-root
63+
# poetry install --no-root
6164
npm install
6265
rm -rf lib/forge-std
6366
forge install --no-git foundry-rs/forge-std@v1.7.3
@@ -83,28 +86,49 @@ function prepare_config() {
8386
mkdir -p ${targetDir} && cd ${targetDir}
8487
cp ${workspace}/keys/password.txt ./
8588
cp ${workspace}/.local/hardforkTime.txt ./
89+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
90+
sed -i -e "s/{{validatorAddr}}/${cons_addr}/g" chaind.sh && rm -f chaind.sh.bak
8691
bbcfee_addrs=${fee_addr}
8792
powers="0x000001d1a94a2000" #2000000000000
8893
mv ${workspace}/.local/bls${i}/bls ./ && rm -rf ${workspace}/.local/bls${i}
8994
vote_addr=0x$(cat ./bls/keystore/*json | jq .pubkey | sed 's/"//g')
9095
echo "${cons_addr},${bbcfee_addrs},${fee_addr},${powers},${vote_addr}" >> ${workspace}/genesis/validators.conf
9196
if [ ${EnableSentryNode} = true ]; then
92-
mkdir -p ${workspace}/.local/sentry${i}
97+
targetDir=${workspace}/.local/sentry${i}
98+
mkdir -p ${targetDir} && cd ${targetDir}
99+
cp ${workspace}/.local/hardforkTime.txt ./
100+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
101+
sed -i -e '/--mine/d' chaind.sh
102+
sed -i -e 's/workdir="validator"/workdir="sentry"/g' chaind.sh
103+
sed -i -e 's/bin="bsc"/bin="sentry"/g' chaind.sh
104+
sed -i -e "s/portInc=0/portInc=2/g" chaind.sh
105+
rm -f chaind.sh.bak
106+
sed -i -e 's/workdir="validator"/workdir="sentry"/g' init.sh
107+
sed -i -e 's/bin="bsc"/bin="sentry"/g' init.sh
108+
rm -f init.sh.bak
109+
mv bsc.service sentry.service
110+
sed -i -e 's/validator/sentry/g' sentry.service
111+
sed -i -e 's/bsc/sentry/g' sentry.service
112+
rm -f sentry.service.bak
93113
fi
94114
done
95115
if [ ${EnableFullNode} = true ]; then
96-
mkdir -p ${workspace}/.local/fullnode0
116+
targetDir=${workspace}/.local/fullnode0
117+
mkdir -p ${targetDir} && cd ${targetDir}
118+
cp ${workspace}/.local/hardforkTime.txt ./
119+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
120+
sed -i -e '/--mine/d' chaind.sh && rm -f chaind.sh.bak
97121
fi
98122
rm -f ${workspace}/.local/hardforkTime.txt
99123

100124
cd ${workspace}/genesis/
101125
git checkout HEAD contracts
102126
sed -i -e 's/alreadyInit = true;/turnLength = 16;alreadyInit = true;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
103127
sed -i -e 's/public onlyCoinbase onlyZeroGasPrice {/public onlyCoinbase onlyZeroGasPrice {if (block.number < 2000) return;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
104-
105-
poetry run python -m scripts.generate generate-validators
106-
poetry run python -m scripts.generate generate-init-holders "${initHolders}"
107-
poetry run python -m scripts.generate dev \
128+
129+
python3 -m scripts.generate generate-validators
130+
python3 -m scripts.generate generate-init-holders "${initHolders}"
131+
python3 -m scripts.generate dev \
108132
--dev-chain-id "${CHAIN_ID}" \
109133
--init-burn-ratio "1000" \
110134
--init-felony-slash-scope "60" \
@@ -142,10 +166,10 @@ function initNetwork() {
142166

143167
init_extra_args=""
144168
if [ ${EnableSentryNode} = true ]; then
145-
init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ports 30411"
169+
init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ips ${sentry_ips_comma}"
146170
fi
147171
if [ ${EnableFullNode} = true ]; then
148-
init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ports 30511"
172+
init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ips ${fullnode_ips_comma}"
149173
fi
150174
if [ "${RegisterNodeID}" = true ]; then
151175
if [ "${EnableSentryNode}" = true ]; then
@@ -161,7 +185,7 @@ function initNetwork() {
161185
init_extra_args="${init_extra_args} --init.evn-validator-whitelist"
162186
fi
163187
fi
164-
${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --config ${workspace}/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
188+
${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --init.ips "${validator_ips_comma}" --config ${workspace}/qa-env-resource/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
165189
rm -f ${workspace}/*bsc.log*
166190
for ((i = 0; i < size; i++)); do
167191
sed -i -e '/"<nil>"/d' ${workspace}/.local/node${i}/config.toml
@@ -175,14 +199,21 @@ function initNetwork() {
175199
rm -f ${workspace}/.local/node${i}/*bsc.log*
176200

177201
if [ ${EnableSentryNode} = true ]; then
202+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/node${i}/config.toml
203+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/node${i}/config.toml
204+
178205
sed -i -e '/"<nil>"/d' ${workspace}/.local/sentry${i}/config.toml
206+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/sentry${i}/config.toml
207+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/sentry${i}/config.toml
179208
initLog=${workspace}/.local/sentry${i}/init.log
180209
${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1
181210
rm -f ${workspace}/.local/sentry${i}/*bsc.log*
182211
fi
183212
done
184213
if [ ${EnableFullNode} = true ]; then
185214
sed -i -e '/"<nil>"/d' ${workspace}/.local/fullnode0/config.toml
215+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/fullnode0/config.toml
216+
sed -i -e 's/:30511/:30311/g' ${workspace}/.local/fullnode0/config.toml
186217
sed -i -e 's/EnableEVNFeatures = true/EnableEVNFeatures = false/g' ${workspace}/.local/fullnode0/config.toml
187218
initLog=${workspace}/.local/fullnode0/init.log
188219
${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1
@@ -274,14 +305,78 @@ function native_start() {
274305
}
275306

276307
function register_stakehub(){
277-
# wait feynman enable
278-
sleep 45
279308
for ((i = 0; i < size; i++));do
280309
${workspace}/create-validator/create-validator --consensus-key-dir ${workspace}/keys/validator${i} --vote-key-dir ${workspace}/keys/bls${i} \
281310
--password-path ${workspace}/keys/password.txt --amount 20001 --validator-desc Val${i} --rpc-url ${RPC_URL}
282311
done
283312
}
284313

314+
function remote_reset_config() {
315+
rm -rf /mnt/efs/${copyDir}/clusterNetwork
316+
cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
317+
ips=(${validator_ips_comma//,/ })
318+
for ((i=0;i<${#ips[@]};i++));do
319+
dst_id=${ips2ids[${ips[i]}]}
320+
if [ ${EnableSentryNode} = true ]; then
321+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/config.toml /server/sentry/"
322+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/chaind.sh /server/sentry/"
323+
fi
324+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/node${i}/config.toml /server/validator/"
325+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/node${i}/chaind.sh /server/validator/"
326+
done
327+
if [ ${EnableFullNode} = true ]; then
328+
fullnode_ips=(${fullnode_ips_comma//,/ })
329+
dst_id=${ips2ids[${fullnode_ips[0]}]}
330+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/fullnode0/config.toml /server/validator/"
331+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/fullnode0/chaind.sh /server/validator/"
332+
fi
333+
}
334+
335+
function remote_start() {
336+
rm -rf /mnt/efs/${copyDir}/clusterNetwork
337+
cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
338+
for dst_id in ${ips2ids[@]}; do
339+
# Always stop sentry nodes, regardless of the target setup.
340+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo service sentry stop"
341+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo service bsc stop"
342+
done
343+
sleep 80
344+
cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
345+
ips=(${validator_ips_comma//,/ })
346+
for ((i=0;i<${#ips[@]};i++));do
347+
dst_id=${ips2ids[${ips[i]}]}
348+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/node${i}/init.sh"
349+
done
350+
if [ ${EnableSentryNode} = true ]; then
351+
sleep 20
352+
for ((i=0;i<${#ips[@]};i++));do
353+
dst_id=${ips2ids[${ips[i]}]}
354+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/init.sh"
355+
done
356+
fi
357+
if [ ${EnableFullNode} = true ]; then
358+
fullnode_ips=(${fullnode_ips_comma//,/ })
359+
dst_id=${ips2ids[${fullnode_ips[0]}]}
360+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/fullnode0/init.sh"
361+
fi
362+
}
363+
364+
function remote_upgrade() {
365+
cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
366+
if [ ${EnableSentryNode} = true ]; then
367+
cp ${workspace}/qa-env-resource/upgrade-single-sentry.sh /mnt/efs/${copyDir}/clusterNetwork/
368+
fi
369+
cp ${workspace}/qa-env-resource/upgrade-single-validator.sh /mnt/efs/${copyDir}/clusterNetwork/
370+
for dst_id in ${ips2ids[@]}; do
371+
if [ ${EnableSentryNode} = true ]; then
372+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
373+
--parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-sentry.sh /tmp/ && sudo bash -x /tmp/upgrade-single-sentry.sh"
374+
fi
375+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
376+
--parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-validator.sh /tmp/ && sudo bash -x /tmp/upgrade-single-validator.sh"
377+
done
378+
}
379+
285380
CMD=$1
286381
ValidatorIdx=$2
287382
case ${CMD} in
@@ -292,7 +387,7 @@ reset)
292387
reset_genesis
293388
prepare_config
294389
initNetwork
295-
native_start
390+
native_start
296391
register_stakehub
297392
;;
298393
stop)
@@ -305,7 +400,30 @@ restart)
305400
exit_previous $ValidatorIdx
306401
native_start $ValidatorIdx
307402
;;
403+
remote_reset)
404+
create_validator
405+
reset_genesis
406+
prepare_config
407+
initNetwork
408+
remote_start
409+
# to prevent stuck
410+
sleep 50
411+
register_stakehub
412+
;;
413+
remote_reset_config)
414+
create_validator
415+
reset_genesis
416+
prepare_config
417+
initNetwork
418+
remote_reset_config
419+
;;
420+
remote_upgrade)
421+
remote_upgrade
422+
;;
423+
register_stakehub)
424+
register_stakehub
425+
;;
308426
*)
309-
echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]"
427+
echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]| remote_reset | remote_upgrade | register_stakehub"
310428
;;
311429
esac

‎qa-env-resource/bsc.service‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[Unit]
2+
Description=bsc
3+
After=network.target
4+
5+
[Service]
6+
Type=simple
7+
User=root
8+
Group=root
9+
ExecStart=/server/validator/chaind.sh -start
10+
ExecReload=/server/validator/chaind.sh -restart
11+
ExecStop=/server/validator/chaind.sh -stop
12+
PrivateTmp=true
13+
LimitNOFILE=10000
14+
StartLimitInterval=0
15+
TimeoutStopSec=120
16+
17+
[Install]
18+
WantedBy=multi-user.target

‎qa-env-resource/chaind.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
export GOGC=200
6+
# default values
7+
workdir="validator"
8+
bin="bsc"
9+
portInc=0
10+
FullImmutabilityThreshold=90000
11+
MinBlocksForBlobRequests=524288
12+
DefaultExtraReserveForBlobRequests=28800
13+
BreatheBlockInterval=600
14+
LAST_FORK_MORE_DELAY=200
15+
16+
function startChaind() {
17+
workspace=/server/${workdir}
18+
19+
PassedForkTime=`cat ${workspace}/hardforkTime.txt|grep passedHardforkTime|awk -F" " '{print $NF}'`
20+
LastHardforkTime=$(expr ${PassedForkTime} + ${LAST_FORK_MORE_DELAY})
21+
initLog=${workspace}/init.log
22+
rialtoHash=`cat ${initLog}|grep "database=chaindata"|awk -F"=" '{print $NF}'|awk -F'"' '{print $1}'`
23+
24+
ip=`ifconfig eth0|grep inet|grep -v inet6 |awk '{ print $2 }'`
25+
sed -i -e "s?FileRoot = \"\"?FileRoot = \"/mnt/efs/${workdir}/${ip}/\"?g" /server/${workdir}/config.toml
26+
mkdir -p /mnt/efs/${workdir}/${ip}
27+
HTTPPort=$((8545 + ${portInc}))
28+
WSPort=${HTTPPort}
29+
MetricsPort=$((6060 + ${portInc}))
30+
PProfPort=$((${MetricsPort} + 1))
31+
${workspace}/${bin} --config ${workspace}/config.toml \
32+
--mine --vote --unlock {{validatorAddr}} --miner.etherbase {{validatorAddr}} --password ${workspace}/password.txt --blspassword ${workspace}/password.txt \
33+
--datadir ${workspace} \
34+
--rpc.allow-unprotected-txs --allow-insecure-unlock \
35+
--ws --ws.addr ${ip} --ws.port ${WSPort} --http --http.addr ${ip} --http.port ${HTTPPort} --http.corsdomain "*" \
36+
--metrics --metrics.addr 0.0.0.0 --metrics.port ${MetricsPort} \
37+
--pprof --pprof.port ${PProfPort} \
38+
--syncmode full --monitor.maliciousvote \
39+
--history.logs.disable \
40+
--cache 10480 \
41+
--rialtohash ${rialtoHash} --override.passedforktime ${PassedForkTime} --override.lorentz ${PassedForkTime} --override.maxwell ${PassedForkTime} --override.fermi ${PassedForkTime} \
42+
--override.osaka ${LastHardforkTime} --override.mendel ${LastHardforkTime} \
43+
--override.immutabilitythreshold ${FullImmutabilityThreshold} --override.breatheblockinterval ${BreatheBlockInterval} \
44+
--override.minforblobrequest ${MinBlocksForBlobRequests} --override.defaultextrareserve ${DefaultExtraReserveForBlobRequests} \
45+
>> /mnt/efs/${workdir}/${ip}/bscnode.log 2>&1
46+
}
47+
48+
function stopChaind() {
49+
pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
50+
if [ -n "$pid" ]; then
51+
kill -TERM $pid
52+
for((i=1;i<=40;i++));
53+
do
54+
pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
55+
if [ -z "$pid" ]; then
56+
break
57+
fi
58+
sleep 10
59+
done
60+
fi
61+
}
62+
63+
CMD=$1
64+
65+
case $CMD in
66+
-start)
67+
echo "start"
68+
startChaind
69+
;;
70+
-stop)
71+
echo "stop"
72+
stopChaind
73+
;;
74+
-restart)
75+
stopChaind
76+
sleep 3
77+
startChaind
78+
;;
79+
*)
80+
echo "Usage: chaind.sh -start | -stop | -restart .Or use systemctl start | stop | restart ${bin}.service "
81+
;;
82+
esac

0 commit comments

Comments
 (0)