Skip to content

Commit a383f70

Browse files
NathanBSCallformless
NathanBSC
authored and
allformless
committed
support QA deploy
1 parent befc8ee commit a383f70

File tree

9 files changed

+390
-20
lines changed

9 files changed

+390
-20
lines changed

‎.env‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ INIT_HOLDER="0x04d63aBCd2b9b1baa327f2Dda0f873F197ccd186"
55
# INIT_HOLDER_PRV="59ba8068eb256d520179e903f43dacf6d8d57d72bd306e1bd603fdb8c8da10e8"
66
RPC_URL="http://127.0.0.1:8545"
77
GENESIS_COMMIT="234e3685ec309624f0fbef41043ae117caebc853" # lorentz commit
8-
PASSED_FORK_DELAY=40
8+
PASSED_FORK_DELAY=600
99
LAST_FORK_MORE_DELAY=10
1010
FullImmutabilityThreshold=2048
1111
MinBlocksForBlobRequests=576
1212
DefaultExtraReserveForBlobRequests=32
1313
BreatheBlockInterval=1200
1414
useLatestBscClient=false
15-
EnableSentryNode=false
16-
EnableFullNode=false
17-
RegisterNodeID=false
18-
EnableEVNWhitelist=false
15+
EnableSentryNode=true
16+
EnableFullNode=true
17+
RegisterNodeID=true
18+
EnableEVNWhitelist=true

‎bsc_cluster.sh‎

Lines changed: 132 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,15 @@ basedir=$(
99
)
1010
workspace=${basedir}
1111
source ${workspace}/.env
12-
size=$((BSC_CLUSTER_SIZE))
12+
source ${workspace}/qa-env-resource/machines_meta.sh # including machine ips and ids, don't upload!!!
13+
validator_ips=(${validator_ips_comma//,/ })
14+
size=${#validator_ips[@]}
1315
stateScheme="hash"
1416
dbEngine="leveldb"
1517
gcmode="full"
1618
sleepBeforeStart=15
1719
sleepAfterStart=10
20+
copyDir="bsc-qa"
1821

1922
# stop geth client
2023
function exit_previous() {
@@ -57,7 +60,7 @@ function reset_genesis() {
5760
mv genesis-template.json.bk genesis-template.json
5861
mv scripts/init_holders.template.bk scripts/init_holders.template
5962

60-
poetry install --no-root
63+
# poetry install --no-root
6164
npm install
6265
rm -rf lib/forge-std
6366
forge install --no-git foundry-rs/forge-std@v1.7.3
@@ -83,28 +86,49 @@ function prepare_config() {
8386
mkdir -p ${targetDir} && cd ${targetDir}
8487
cp ${workspace}/keys/password.txt ./
8588
cp ${workspace}/.local/hardforkTime.txt ./
89+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
90+
sed -i -e "s/{{validatorAddr}}/${cons_addr}/g" chaind.sh && rm -f chaind.sh.bak
8691
bbcfee_addrs=${fee_addr}
8792
powers="0x000001d1a94a2000" #2000000000000
8893
mv ${workspace}/.local/bls${i}/bls ./ && rm -rf ${workspace}/.local/bls${i}
8994
vote_addr=0x$(cat ./bls/keystore/*json | jq .pubkey | sed 's/"//g')
9095
echo "${cons_addr},${bbcfee_addrs},${fee_addr},${powers},${vote_addr}" >> ${workspace}/genesis/validators.conf
9196
if [ ${EnableSentryNode} = true ]; then
92-
mkdir -p ${workspace}/.local/sentry${i}
97+
targetDir=${workspace}/.local/sentry${i}
98+
mkdir -p ${targetDir} && cd ${targetDir}
99+
cp ${workspace}/.local/hardforkTime.txt ./
100+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
101+
sed -i -e '/--mine/d' chaind.sh
102+
sed -i -e 's/workdir="validator"/workdir="sentry"/g' chaind.sh
103+
sed -i -e 's/bin="bsc"/bin="sentry"/g' chaind.sh
104+
sed -i -e "s/portInc=0/portInc=2/g" chaind.sh
105+
rm -f chaind.sh.bak
106+
sed -i -e 's/workdir="validator"/workdir="sentry"/g' init.sh
107+
sed -i -e 's/bin="bsc"/bin="sentry"/g' init.sh
108+
rm -f init.sh.bak
109+
mv bsc.service sentry.service
110+
sed -i -e 's/validator/sentry/g' sentry.service
111+
sed -i -e 's/bsc/sentry/g' sentry.service
112+
rm -f sentry.service.bak
93113
fi
94114
done
95115
if [ ${EnableFullNode} = true ]; then
96-
mkdir -p ${workspace}/.local/fullnode0
116+
targetDir=${workspace}/.local/fullnode0
117+
mkdir -p ${targetDir} && cd ${targetDir}
118+
cp ${workspace}/.local/hardforkTime.txt ./
119+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
120+
sed -i -e '/--mine/d' chaind.sh && rm -f chaind.sh.bak
97121
fi
98122
rm -f ${workspace}/.local/hardforkTime.txt
99123

100124
cd ${workspace}/genesis/
101125
git checkout HEAD contracts
102126
sed -i -e 's/alreadyInit = true;/turnLength = 16;alreadyInit = true;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
103127
sed -i -e 's/public onlyCoinbase onlyZeroGasPrice {/public onlyCoinbase onlyZeroGasPrice {if (block.number < 300) return;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
104-
105-
poetry run python -m scripts.generate generate-validators
106-
poetry run python -m scripts.generate generate-init-holders "${initHolders}"
107-
poetry run python -m scripts.generate dev \
128+
129+
python3 -m scripts.generate generate-validators
130+
python3 -m scripts.generate generate-init-holders "${initHolders}"
131+
python3 -m scripts.generate dev \
108132
--dev-chain-id "${CHAIN_ID}" \
109133
--init-burn-ratio "1000" \
110134
--init-felony-slash-scope "60" \
@@ -121,6 +145,7 @@ function prepare_config() {
121145
--governor-protector "${INIT_HOLDER}" \
122146
--init-minimal-delay "1 minutes" \
123147
--token-recover-portal-protector "${INIT_HOLDER}"
148+
#cp genesis-dev.json genesis.json
124149
}
125150

126151
function initNetwork() {
@@ -141,10 +166,10 @@ function initNetwork() {
141166

142167
init_extra_args=""
143168
if [ ${EnableSentryNode} = true ]; then
144-
init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ports 30411"
169+
init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ips ${sentry_ips_comma}"
145170
fi
146171
if [ ${EnableFullNode} = true ]; then
147-
init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ports 30511"
172+
init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ips ${fullnode_ips_comma}"
148173
fi
149174
if [ "${RegisterNodeID}" = true ]; then
150175
if [ "${EnableSentryNode}" = true ]; then
@@ -160,7 +185,7 @@ function initNetwork() {
160185
init_extra_args="${init_extra_args} --init.evn-validator-whitelist"
161186
fi
162187
fi
163-
${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --config ${workspace}/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
188+
${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --init.ips "${validator_ips_comma}" --config ${workspace}/qa-env-resource/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
164189
rm -f ${workspace}/*bsc.log*
165190
for ((i = 0; i < size; i++)); do
166191
sed -i -e '/"<nil>"/d' ${workspace}/.local/node${i}/config.toml
@@ -174,14 +199,21 @@ function initNetwork() {
174199
rm -f ${workspace}/.local/node${i}/*bsc.log*
175200

176201
if [ ${EnableSentryNode} = true ]; then
202+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/node${i}/config.toml
203+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/node${i}/config.toml
204+
177205
sed -i -e '/"<nil>"/d' ${workspace}/.local/sentry${i}/config.toml
206+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/sentry${i}/config.toml
207+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/sentry${i}/config.toml
178208
initLog=${workspace}/.local/sentry${i}/init.log
179209
${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1
180210
rm -f ${workspace}/.local/sentry${i}/*bsc.log*
181211
fi
182212
done
183213
if [ ${EnableFullNode} = true ]; then
184214
sed -i -e '/"<nil>"/d' ${workspace}/.local/fullnode0/config.toml
215+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/fullnode0/config.toml
216+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/fullnode0/config.toml
185217
sed -i -e 's/EnableEVNFeatures = true/EnableEVNFeatures = false/g' ${workspace}/.local/fullnode0/config.toml
186218
initLog=${workspace}/.local/fullnode0/init.log
187219
${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1
@@ -264,14 +296,74 @@ function native_start() {
264296
}
265297

266298
function register_stakehub(){
267-
# wait feynman enable
268-
sleep 45
269299
for ((i = 0; i < size; i++));do
270300
${workspace}/create-validator/create-validator --consensus-key-dir ${workspace}/keys/validator${i} --vote-key-dir ${workspace}/keys/bls${i} \
271301
--password-path ${workspace}/keys/password.txt --amount 20001 --validator-desc Val${i} --rpc-url ${RPC_URL}
272302
done
273303
}
274304

305+
function remote_reset_config() {
306+
rm -rf /mnt/efs/${copyDir}/clusterNetwork
307+
cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
308+
ips=(${validator_ips_comma//,/ })
309+
for ((i=0;i<${#ips[@]};i++));do
310+
dst_id=${ips2ids[${ips[i]}]}
311+
if [ ${EnableSentryNode} = true ]; then
312+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/config.toml /server/sentry/"
313+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/chaind.sh /server/sentry/"
314+
fi
315+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/node${i}/config.toml /server/validator/"
316+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/node${i}/chaind.sh /server/validator/"
317+
done
318+
if [ ${EnableFullNode} = true ]; then
319+
fullnode_ips=(${fullnode_ips_comma//,/ })
320+
dst_id=${ips2ids[${fullnode_ips[0]}]}
321+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/fullnode0/config.toml /server/validator/"
322+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo \cp -f /mnt/efs/${copyDir}/clusterNetwork/fullnode0/chaind.sh /server/validator/"
323+
fi
324+
}
325+
326+
function remote_start() {
327+
rm -rf /mnt/efs/${copyDir}/clusterNetwork
328+
cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
329+
for dst_id in ${ips2ids[@]}; do
330+
# Always stop sentry nodes, regardless of the target setup.
331+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo service sentry stop"
332+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo service bsc stop"
333+
done
334+
sleep 100
335+
cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
336+
ips=(${validator_ips_comma//,/ })
337+
for ((i=0;i<${#ips[@]};i++));do
338+
dst_id=${ips2ids[${ips[i]}]}
339+
if [ ${EnableSentryNode} = true ]; then
340+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/init.sh"
341+
fi
342+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/node${i}/init.sh"
343+
done
344+
if [ ${EnableFullNode} = true ]; then
345+
fullnode_ips=(${fullnode_ips_comma//,/ })
346+
dst_id=${ips2ids[${fullnode_ips[0]}]}
347+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/fullnode0/init.sh"
348+
fi
349+
}
350+
351+
function remote_upgrade() {
352+
cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
353+
if [ ${EnableSentryNode} = true ]; then
354+
cp ${workspace}/qa-env-resource/upgrade-single-sentry.sh /mnt/efs/${copyDir}/clusterNetwork/
355+
fi
356+
cp ${workspace}/qa-env-resource/upgrade-single-validator.sh /mnt/efs/${copyDir}/clusterNetwork/
357+
for dst_id in ${ips2ids[@]}; do
358+
if [ ${EnableSentryNode} = true ]; then
359+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
360+
--parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-sentry.sh /tmp/ && sudo bash -x /tmp/upgrade-single-sentry.sh"
361+
fi
362+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
363+
--parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-validator.sh /tmp/ && sudo bash -x /tmp/upgrade-single-validator.sh"
364+
done
365+
}
366+
275367
CMD=$1
276368
ValidatorIdx=$2
277369
case ${CMD} in
@@ -282,7 +374,7 @@ reset)
282374
reset_genesis
283375
prepare_config
284376
initNetwork
285-
native_start
377+
native_start
286378
register_stakehub
287379
;;
288380
stop)
@@ -295,7 +387,32 @@ restart)
295387
exit_previous $ValidatorIdx
296388
native_start $ValidatorIdx
297389
;;
390+
remote_reset)
391+
create_validator
392+
reset_genesis
393+
prepare_config
394+
initNetwork
395+
remote_start
396+
# to prevent stuck
397+
sleep 50
398+
remote_upgrade
399+
sleep 50 # wait feynman enable
400+
register_stakehub
401+
;;
402+
remote_reset_config)
403+
create_validator
404+
reset_genesis
405+
prepare_config
406+
initNetwork
407+
remote_reset_config
408+
;;
409+
remote_upgrade)
410+
remote_upgrade
411+
;;
412+
register_stakehub)
413+
register_stakehub
414+
;;
298415
*)
299-
echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]"
416+
echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]| remote_reset | remote_upgrade | register_stakehub"
300417
;;
301418
esac

‎qa-env-resource/bsc.service‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
[Unit]
2+
Description=bsc
3+
After=network.target
4+
5+
[Service]
6+
Type=simple
7+
User=root
8+
Group=root
9+
ExecStart=/server/validator/chaind.sh -start
10+
ExecReload=/server/validator/chaind.sh -restart
11+
ExecStop=/server/validator/chaind.sh -stop
12+
PrivateTmp=true
13+
LimitNOFILE=10000
14+
StartLimitInterval=0
15+
TimeoutStopSec=120
16+
17+
[Install]
18+
WantedBy=multi-user.target

‎qa-env-resource/chaind.sh‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
export GOGC=200
6+
# default values
7+
workdir="validator"
8+
bin="bsc"
9+
portInc=0
10+
FullImmutabilityThreshold=90000
11+
MinBlocksForBlobRequests=524288
12+
DefaultExtraReserveForBlobRequests=28800
13+
BreatheBlockInterval=600
14+
LAST_FORK_MORE_DELAY=200
15+
16+
function startChaind() {
17+
workspace=/server/${workdir}
18+
19+
PassedForkTime=`cat ${workspace}/hardforkTime.txt|grep passedHardforkTime|awk -F" " '{print $NF}'`
20+
LastHardforkTime=$(expr ${PassedForkTime} + ${LAST_FORK_MORE_DELAY})
21+
initLog=${workspace}/init.log
22+
rialtoHash=`cat ${initLog}|grep "database=chaindata"|awk -F"=" '{print $NF}'|awk -F'"' '{print $1}'`
23+
24+
ip=`ifconfig eth0|grep inet|grep -v inet6 |awk '{ print $2 }'`
25+
sed -i -e "s?FileRoot = \"\"?FileRoot = \"/mnt/efs/${workdir}/${ip}/\"?g" /server/${workdir}/config.toml
26+
mkdir -p /mnt/efs/${workdir}/${ip}
27+
HTTPPort=$((8545 + ${portInc}))
28+
WSPort=${HTTPPort}
29+
MetricsPort=$((6060 + ${portInc}))
30+
PProfPort=$((${MetricsPort} + 1))
31+
${workspace}/${bin} --config ${workspace}/config.toml \
32+
--mine --vote --unlock {{validatorAddr}} --miner.etherbase {{validatorAddr}} --password ${workspace}/password.txt --blspassword ${workspace}/password.txt \
33+
--datadir ${workspace} \
34+
--rpc.allow-unprotected-txs --allow-insecure-unlock \
35+
--ws --ws.addr ${ip} --ws.port ${WSPort} --http --http.addr ${ip} --http.port ${HTTPPort} --http.corsdomain "*" \
36+
--metrics --metrics.addr 0.0.0.0 --metrics.port ${MetricsPort} \
37+
--pprof --pprof.port ${PProfPort} \
38+
--syncmode full --monitor.maliciousvote \
39+
--cache 10480 \
40+
--rialtohash ${rialtoHash} --override.passedforktime ${PassedForkTime} --override.lorentz ${PassedForkTime} --override.maxwell ${PassedForkTime} --override.fermi ${LastHardforkTime} \
41+
--override.immutabilitythreshold ${FullImmutabilityThreshold} --override.breatheblockinterval ${BreatheBlockInterval} \
42+
--override.minforblobrequest ${MinBlocksForBlobRequests} --override.defaultextrareserve ${DefaultExtraReserveForBlobRequests} \
43+
>> /mnt/efs/${workdir}/${ip}/bscnode.log 2>&1
44+
}
45+
46+
function stopChaind() {
47+
pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
48+
if [ -n "$pid" ]; then
49+
kill -TERM $pid
50+
for((i=1;i<=40;i++));
51+
do
52+
pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
53+
if [ -z "$pid" ]; then
54+
break
55+
fi
56+
sleep 10
57+
done
58+
fi
59+
}
60+
61+
CMD=$1
62+
63+
case $CMD in
64+
-start)
65+
echo "start"
66+
startChaind
67+
;;
68+
-stop)
69+
echo "stop"
70+
stopChaind
71+
;;
72+
-restart)
73+
stopChaind
74+
sleep 3
75+
startChaind
76+
;;
77+
*)
78+
echo "Usage: chaind.sh -start | -stop | -restart .Or use systemctl start | stop | restart ${bin}.service "
79+
;;
80+
esac

0 commit comments

Comments
 (0)