Skip to content

Commit 228a594

Browse files
NathanBSCallformless
NathanBSC
authored and
allformless
committed
support QA deploy
1 parent ba0fdbd commit 228a594

File tree

9 files changed

+392
-20
lines changed

9 files changed

+392
-20
lines changed

‎.env‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ INIT_HOLDER="0x04d63aBCd2b9b1baa327f2Dda0f873F197ccd186"
55
# INIT_HOLDER_PRV="59ba8068eb256d520179e903f43dacf6d8d57d72bd306e1bd603fdb8c8da10e8"
66
RPC_URL="http://127.0.0.1:8545"
77
GENESIS_COMMIT="34618f607f8356cf147dde6a69fae150bd53d5bf" # fermi commit
8-
PASSED_FORK_DELAY=40
8+
PASSED_FORK_DELAY=600
99
LAST_FORK_MORE_DELAY=10
1010
FullImmutabilityThreshold=2048
1111
MinBlocksForBlobRequests=576
1212
DefaultExtraReserveForBlobRequests=32
1313
BreatheBlockInterval=1200
1414
useLatestBscClient=false
15-
EnableSentryNode=false
16-
EnableFullNode=false
17-
RegisterNodeID=false
18-
EnableEVNWhitelist=false
15+
EnableSentryNode=true
16+
EnableFullNode=true
17+
RegisterNodeID=true
18+
EnableEVNWhitelist=true

‎bsc_cluster.sh‎

Lines changed: 133 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,15 @@ basedir=$(
99
)
1010
workspace=${basedir}
1111
source ${workspace}/.env
12-
size=$((BSC_CLUSTER_SIZE))
12+
source ${workspace}/qa-env-resource/machines_meta.sh # including machine ips and ids, don't upload!!!
13+
validator_ips=(${validator_ips_comma//,/ })
14+
size=${#validator_ips[@]}
1315
stateScheme="hash"
1416
dbEngine="leveldb"
1517
gcmode="full"
1618
sleepBeforeStart=15
1719
sleepAfterStart=10
20+
copyDir="bsc-qa"
1821

1922
# stop geth client
2023
function exit_previous() {
@@ -57,7 +60,7 @@ function reset_genesis() {
5760
mv genesis-template.json.bk genesis-template.json
5861
mv scripts/init_holders.template.bk scripts/init_holders.template
5962

60-
poetry install --no-root
63+
# poetry install --no-root
6164
npm install
6265
rm -rf lib/forge-std
6366
forge install --no-git foundry-rs/forge-std@v1.7.3
@@ -83,28 +86,49 @@ function prepare_config() {
8386
mkdir -p ${targetDir} && cd ${targetDir}
8487
cp ${workspace}/keys/password.txt ./
8588
cp ${workspace}/.local/hardforkTime.txt ./
89+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
90+
sed -i -e "s/{{validatorAddr}}/${cons_addr}/g" chaind.sh && rm -f chaind.sh.bak
8691
bbcfee_addrs=${fee_addr}
8792
powers="0x000001d1a94a2000" #2000000000000
8893
mv ${workspace}/.local/bls${i}/bls ./ && rm -rf ${workspace}/.local/bls${i}
8994
vote_addr=0x$(cat ./bls/keystore/*json | jq .pubkey | sed 's/"//g')
9095
echo "${cons_addr},${bbcfee_addrs},${fee_addr},${powers},${vote_addr}" >> ${workspace}/genesis/validators.conf
9196
if [ ${EnableSentryNode} = true ]; then
92-
mkdir -p ${workspace}/.local/sentry${i}
97+
targetDir=${workspace}/.local/sentry${i}
98+
mkdir -p ${targetDir} && cd ${targetDir}
99+
cp ${workspace}/.local/hardforkTime.txt ./
100+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
101+
sed -i -e '/--mine/d' chaind.sh
102+
sed -i -e 's/workdir="validator"/workdir="sentry"/g' chaind.sh
103+
sed -i -e 's/bin="bsc"/bin="sentry"/g' chaind.sh
104+
sed -i -e "s/portInc=0/portInc=2/g" chaind.sh
105+
rm -f chaind.sh.bak
106+
sed -i -e 's/workdir="validator"/workdir="sentry"/g' init.sh
107+
sed -i -e 's/bin="bsc"/bin="sentry"/g' init.sh
108+
rm -f init.sh.bak
109+
mv bsc.service sentry.service
110+
sed -i -e 's/validator/sentry/g' sentry.service
111+
sed -i -e 's/bsc/sentry/g' sentry.service
112+
rm -f sentry.service.bak
93113
fi
94114
done
95115
if [ ${EnableFullNode} = true ]; then
96-
mkdir -p ${workspace}/.local/fullnode0
116+
targetDir=${workspace}/.local/fullnode0
117+
mkdir -p ${targetDir} && cd ${targetDir}
118+
cp ${workspace}/.local/hardforkTime.txt ./
119+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
120+
sed -i -e '/--mine/d' chaind.sh && rm -f chaind.sh.bak
97121
fi
98122
rm -f ${workspace}/.local/hardforkTime.txt
99123

100124
cd ${workspace}/genesis/
101125
git checkout HEAD contracts
102126
sed -i -e 's/alreadyInit = true;/turnLength = 16;alreadyInit = true;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
103127
sed -i -e 's/public onlyCoinbase onlyZeroGasPrice {/public onlyCoinbase onlyZeroGasPrice {if (block.number < 2000) return;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
104-
105-
poetry run python -m scripts.generate generate-validators
106-
poetry run python -m scripts.generate generate-init-holders "${initHolders}"
107-
poetry run python -m scripts.generate dev \
128+
129+
python3 -m scripts.generate generate-validators
130+
python3 -m scripts.generate generate-init-holders "${initHolders}"
131+
python3 -m scripts.generate dev \
108132
--dev-chain-id "${CHAIN_ID}" \
109133
--init-burn-ratio "1000" \
110134
--init-felony-slash-scope "60" \
@@ -142,10 +166,10 @@ function initNetwork() {
142166

143167
init_extra_args=""
144168
if [ ${EnableSentryNode} = true ]; then
145-
init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ports 30411"
169+
init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ips ${sentry_ips_comma}"
146170
fi
147171
if [ ${EnableFullNode} = true ]; then
148-
init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ports 30511"
172+
init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ips ${fullnode_ips_comma}"
149173
fi
150174
if [ "${RegisterNodeID}" = true ]; then
151175
if [ "${EnableSentryNode}" = true ]; then
@@ -161,7 +185,7 @@ function initNetwork() {
161185
init_extra_args="${init_extra_args} --init.evn-validator-whitelist"
162186
fi
163187
fi
164-
${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --config ${workspace}/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
188+
${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --init.ips "${validator_ips_comma}" --config ${workspace}/qa-env-resource/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
165189
rm -f ${workspace}/*bsc.log*
166190
for ((i = 0; i < size; i++)); do
167191
sed -i -e '/"<nil>"/d' ${workspace}/.local/node${i}/config.toml
@@ -175,14 +199,21 @@ function initNetwork() {
175199
rm -f ${workspace}/.local/node${i}/*bsc.log*
176200

177201
if [ ${EnableSentryNode} = true ]; then
202+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/node${i}/config.toml
203+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/node${i}/config.toml
204+
178205
sed -i -e '/"<nil>"/d' ${workspace}/.local/sentry${i}/config.toml
206+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/sentry${i}/config.toml
207+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/sentry${i}/config.toml
179208
initLog=${workspace}/.local/sentry${i}/init.log
180209
${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1
181210
rm -f ${workspace}/.local/sentry${i}/*bsc.log*
182211
fi
183212
done
184213
if [ ${EnableFullNode} = true ]; then
185214
sed -i -e '/"<nil>"/d' ${workspace}/.local/fullnode0/config.toml
215+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/fullnode0/config.toml
216+
sed -i -e 's/:30511/:30311/g' ${workspace}/.local/fullnode0/config.toml
186217
sed -i -e 's/EnableEVNFeatures = true/EnableEVNFeatures = false/g' ${workspace}/.local/fullnode0/config.toml
187218
initLog=${workspace}/.local/fullnode0/init.log
188219
${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1
@@ -272,14 +303,78 @@ function native_start() {
272303
}
273304

274305
function register_stakehub(){
275-
# wait feynman enable
276-
sleep 45
277306
for ((i = 0; i < size; i++));do
278307
${workspace}/create-validator/create-validator --consensus-key-dir ${workspace}/keys/validator${i} --vote-key-dir ${workspace}/keys/bls${i} \
279308
--password-path ${workspace}/keys/password.txt --amount 20001 --validator-desc Val${i} --rpc-url ${RPC_URL}
280309
done
281310
}
282311

312+
function remote_reset_config() {
313+
rm -rf /mnt/efs/${copyDir}/clusterNetwork
314+
cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
315+
ips=(${validator_ips_comma//,/ })
316+
for ((i=0;i<${#ips[@]};i++));do
317+
dst_id=${ips2ids[${ips[i]}]}
318+
if [ ${EnableSentryNode} = true ]; then
319+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/config.toml /server/sentry/"
320+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/chaind.sh /server/sentry/"
321+
fi
322+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/node${i}/config.toml /server/validator/"
323+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/node${i}/chaind.sh /server/validator/"
324+
done
325+
if [ ${EnableFullNode} = true ]; then
326+
fullnode_ips=(${fullnode_ips_comma//,/ })
327+
dst_id=${ips2ids[${fullnode_ips[0]}]}
328+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/fullnode0/config.toml /server/validator/"
329+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/fullnode0/chaind.sh /server/validator/"
330+
fi
331+
}
332+
333+
function remote_start() {
334+
rm -rf /mnt/efs/${copyDir}/clusterNetwork
335+
cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
336+
for dst_id in ${ips2ids[@]}; do
337+
# Always stop sentry nodes, regardless of the target setup.
338+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo service sentry stop"
339+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo service bsc stop"
340+
done
341+
sleep 80
342+
cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
343+
ips=(${validator_ips_comma//,/ })
344+
for ((i=0;i<${#ips[@]};i++));do
345+
dst_id=${ips2ids[${ips[i]}]}
346+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/node${i}/init.sh"
347+
done
348+
if [ ${EnableSentryNode} = true ]; then
349+
sleep 20
350+
for ((i=0;i<${#ips[@]};i++));do
351+
dst_id=${ips2ids[${ips[i]}]}
352+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/init.sh"
353+
done
354+
fi
355+
if [ ${EnableFullNode} = true ]; then
356+
fullnode_ips=(${fullnode_ips_comma//,/ })
357+
dst_id=${ips2ids[${fullnode_ips[0]}]}
358+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/fullnode0/init.sh"
359+
fi
360+
}
361+
362+
function remote_upgrade() {
363+
cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
364+
if [ ${EnableSentryNode} = true ]; then
365+
cp ${workspace}/qa-env-resource/upgrade-single-sentry.sh /mnt/efs/${copyDir}/clusterNetwork/
366+
fi
367+
cp ${workspace}/qa-env-resource/upgrade-single-validator.sh /mnt/efs/${copyDir}/clusterNetwork/
368+
for dst_id in ${ips2ids[@]}; do
369+
if [ ${EnableSentryNode} = true ]; then
370+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
371+
--parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-sentry.sh /tmp/ && sudo bash -x /tmp/upgrade-single-sentry.sh"
372+
fi
373+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
374+
--parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-validator.sh /tmp/ && sudo bash -x /tmp/upgrade-single-validator.sh"
375+
done
376+
}
377+
283378
CMD=$1
284379
ValidatorIdx=$2
285380
case ${CMD} in
@@ -290,7 +385,7 @@ reset)
290385
reset_genesis
291386
prepare_config
292387
initNetwork
293-
native_start
388+
native_start
294389
register_stakehub
295390
;;
296391
stop)
@@ -303,7 +398,30 @@ restart)
303398
exit_previous $ValidatorIdx
304399
native_start $ValidatorIdx
305400
;;
401+
remote_reset)
402+
create_validator
403+
reset_genesis
404+
prepare_config
405+
initNetwork
406+
remote_start
407+
# to prevent stuck
408+
sleep 50
409+
register_stakehub
410+
;;
411+
remote_reset_config)
412+
create_validator
413+
reset_genesis
414+
prepare_config
415+
initNetwork
416+
remote_reset_config
417+
;;
418+
remote_upgrade)
419+
remote_upgrade
420+
;;
421+
register_stakehub)
422+
register_stakehub
423+
;;
306424
*)
307-
echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]"
425+
echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]| remote_reset | remote_upgrade | register_stakehub"
308426
;;
309427
esac

‎qa-env-resource/bsc.service‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[Unit]
2+
Description=bsc
3+
After=network.target
4+
5+
[Service]
6+
Type=simple
7+
User=root
8+
Group=root
9+
ExecStart=/server/validator/chaind.sh -start
10+
ExecReload=/server/validator/chaind.sh -restart
11+
ExecStop=/server/validator/chaind.sh -stop
12+
PrivateTmp=true
13+
LimitNOFILE=10000
14+
StartLimitInterval=0
15+
TimeoutStopSec=120
16+
17+
[Install]
18+
WantedBy=multi-user.target

‎qa-env-resource/chaind.sh‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
export GOGC=200
6+
# default values
7+
workdir="validator"
8+
bin="bsc"
9+
portInc=0
10+
FullImmutabilityThreshold=90000
11+
MinBlocksForBlobRequests=524288
12+
DefaultExtraReserveForBlobRequests=28800
13+
BreatheBlockInterval=600
14+
LAST_FORK_MORE_DELAY=200
15+
16+
function startChaind() {
17+
workspace=/server/${workdir}
18+
19+
PassedForkTime=`cat ${workspace}/hardforkTime.txt|grep passedHardforkTime|awk -F" " '{print $NF}'`
20+
LastHardforkTime=$(expr ${PassedForkTime} + ${LAST_FORK_MORE_DELAY})
21+
initLog=${workspace}/init.log
22+
rialtoHash=`cat ${initLog}|grep "database=chaindata"|awk -F"=" '{print $NF}'|awk -F'"' '{print $1}'`
23+
24+
ip=`ifconfig eth0|grep inet|grep -v inet6 |awk '{ print $2 }'`
25+
sed -i -e "s?FileRoot = \"\"?FileRoot = \"/mnt/efs/${workdir}/${ip}/\"?g" /server/${workdir}/config.toml
26+
mkdir -p /mnt/efs/${workdir}/${ip}
27+
HTTPPort=$((8545 + ${portInc}))
28+
WSPort=${HTTPPort}
29+
MetricsPort=$((6060 + ${portInc}))
30+
PProfPort=$((${MetricsPort} + 1))
31+
${workspace}/${bin} --config ${workspace}/config.toml \
32+
--mine --vote --unlock {{validatorAddr}} --miner.etherbase {{validatorAddr}} --password ${workspace}/password.txt --blspassword ${workspace}/password.txt \
33+
--datadir ${workspace} \
34+
--rpc.allow-unprotected-txs --allow-insecure-unlock \
35+
--ws --ws.addr ${ip} --ws.port ${WSPort} --http --http.addr ${ip} --http.port ${HTTPPort} --http.corsdomain "*" \
36+
--metrics --metrics.addr 0.0.0.0 --metrics.port ${MetricsPort} \
37+
--pprof --pprof.port ${PProfPort} \
38+
--syncmode full --monitor.maliciousvote \
39+
--history.logs.disable \
40+
--cache 10480 \
41+
--rialtohash ${rialtoHash} --override.passedforktime ${PassedForkTime} --override.lorentz ${PassedForkTime} --override.maxwell ${PassedForkTime} --override.fermi ${LastHardforkTime} \
42+
--override.immutabilitythreshold ${FullImmutabilityThreshold} --override.breatheblockinterval ${BreatheBlockInterval} \
43+
--override.minforblobrequest ${MinBlocksForBlobRequests} --override.defaultextrareserve ${DefaultExtraReserveForBlobRequests} \
44+
>> /mnt/efs/${workdir}/${ip}/bscnode.log 2>&1
45+
}
46+
47+
function stopChaind() {
48+
pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
49+
if [ -n "$pid" ]; then
50+
kill -TERM $pid
51+
for((i=1;i<=40;i++));
52+
do
53+
pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
54+
if [ -z "$pid" ]; then
55+
break
56+
fi
57+
sleep 10
58+
done
59+
fi
60+
}
61+
62+
CMD=$1
63+
64+
case $CMD in
65+
-start)
66+
echo "start"
67+
startChaind
68+
;;
69+
-stop)
70+
echo "stop"
71+
stopChaind
72+
;;
73+
-restart)
74+
stopChaind
75+
sleep 3
76+
startChaind
77+
;;
78+
*)
79+
echo "Usage: chaind.sh -start | -stop | -restart .Or use systemctl start | stop | restart ${bin}.service "
80+
;;
81+
esac

0 commit comments

Comments
 (0)