Skip to content

Commit

Permalink
Support create test CVM for lighthouse. v1.0.303
Browse files Browse the repository at this point in the history
  • Loading branch information
winlinvip committed Aug 9, 2023
1 parent f195602 commit e3d99e8
Show file tree
Hide file tree
Showing 7 changed files with 146 additions and 33 deletions.
23 changes: 16 additions & 7 deletions DEVELOPER.md
Original file line number Diff line number Diff line change
Expand Up @@ -302,10 +302,10 @@ Create a CVM instance:

```bash
rm -f /tmp/lh-*.txt &&
VM_TOKEN="ABCabc$(date +%s)" && echo "$VM_TOKEN" >/tmp/lh-token.txt &&
VM_TOKEN=$(openssl rand -base64 16 | tr -dc 'a-zA-Z0-9' | head -c 16) && echo "$VM_TOKEN" >/tmp/lh-token.txt &&
VM_TOKEN=$VM_TOKEN bash scripts/tools/tencent-cloud/helper.sh create-cvm.py 2>/tmp/lh-instance.txt && VM_INSTANCE=$(cat /tmp/lh-instance.txt) &&
bash scripts/tools/tencent-cloud/helper.sh query-cvm-ip.py --instance $VM_INSTANCE 2>/tmp/lh-ip.txt && VM_IP=$(cat /tmp/lh-ip.txt) &&
echo "Instance: $VM_INSTANCE, IP: $VM_IP"
echo "Instance: $VM_INSTANCE, IP: ubuntu@$VM_IP"
```

Run blueprint script:
Expand All @@ -314,18 +314,27 @@ Run blueprint script:
VM_IP=$(cat /tmp/lh-ip.txt) && VM_TOKEN=$(cat /tmp/lh-token.txt) && VM_INSTANCE=$(cat /tmp/lh-instance.txt) &&
bash scripts/setup-lighthouse/build.sh --ip $VM_IP --os ubuntu --user ubuntu --password $VM_TOKEN &&
bash scripts/tools/tencent-cloud/helper.sh create-image.py --instance $VM_INSTANCE 2>/tmp/lh-image.txt && VM_IMAGE=$(cat /tmp/lh-image.txt) &&
bash scripts/tools/tencent-cloud/helper.sh share-image --instance $VM_IMAGE &&
bash scripts/tools/tencent-cloud/helper.sh share-image.py --image $VM_IMAGE &&
echo "Image: $VM_IMAGE created and shared."
```

> Note: We always use the `1.0.0.sh` for SRS Cloud lighthouse.
Next, create a test CVM instance with the image:

```bash
TEST_TOKEN=$(openssl rand -base64 16 | tr -dc 'a-zA-Z0-9' | head -c 16) && VM_IMAGE=$(cat /tmp/lh-image.txt) &&
VM_TOKEN=$TEST_TOKEN bash scripts/tools/tencent-cloud/helper.sh create-verify-cvm.py --image $VM_IMAGE 2>/tmp/lh-test.txt && TEST_INSTANCE=$(cat /tmp/lh-test.txt) &&
bash scripts/tools/tencent-cloud/helper.sh query-cvm-ip.py --instance $TEST_INSTANCE 2>/tmp/lh-ip2.txt && TEST_IP=$(cat /tmp/lh-ip2.txt) &&
echo "Instance: $TEST_INSTANCE, IP: ubuntu@$TEST_IP, Password: $TEST_TOKEN" &&
echo "http://$TEST_IP"
```

Then run the script to remove all the CVM, disk images, and snapshots:

```bash
VM_INSTANCE=$(cat /tmp/lh-instance.txt) && VM_IMAGE=$(cat /tmp/lh-image.txt) &&
(cd scripts/tools/tencent-cloud && source venv/bin/activate && python remove-cvm.py --instance $VM_INSTANCE || echo OK) &&
(cd scripts/tools/tencent-cloud && source venv/bin/activate && python remove-image.py --instance $VM_IMAGE || echo OK) &&
VM_INSTANCE=$(cat /tmp/lh-instance.txt) && VM_IMAGE=$(cat /tmp/lh-image.txt) && TEST_INSTANCE=$(cat /tmp/lh-test.txt) &&
(bash scripts/tools/tencent-cloud/helper.sh remove-cvm.py --instance $TEST_INSTANCE || echo OK) &&
(bash scripts/tools/tencent-cloud/helper.sh remove-cvm.py --instance $VM_INSTANCE || echo OK) &&
(bash scripts/tools/tencent-cloud/helper.sh remove-image.py --image $VM_IMAGE || echo OK) &&
echo "Cleanup Instance: $VM_INSTANCE, Image: $VM_IMAGE OK."
```

Expand Down
2 changes: 1 addition & 1 deletion scripts/setup-lighthouse/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ echo " tar xf $tgzFile"
echo " bash ~/$SRS_NAME/scripts/setup-lighthouse/setup_lighthouse.sh"

$sshCmd -t $user@$ip "
tar xf $tgzFile && \
rm -rf $SRS_NAME && tar xf $tgzFile && \
sudo bash $SRS_NAME/scripts/setup-lighthouse/setup_lighthouse.sh &&
if [[ $cleanup == yes ]]; then
sudo bash $SRS_NAME/scripts/setup-lighthouse/post_build.sh &&
Expand Down
5 changes: 2 additions & 3 deletions scripts/setup-lighthouse/setup_lighthouse.sh
Original file line number Diff line number Diff line change
Expand Up @@ -85,11 +85,10 @@ update_sysctl net.core.wmem_max 16777216
update_sysctl net.core.wmem_default 16777216

# Install files to lighthouse directory.
rm -rf ${SRS_HOME} && mkdir -p $SRS_HOME/mgmt &&
cp -r ${SOURCE}/usr ${SRS_HOME}/usr &&
cp ${SOURCE}/LICENSE ${SRS_HOME}/LICENSE &&
cp ${SOURCE}/README.md ${SRS_HOME}/README.md &&
cp ${SOURCE}/mgmt/bootstrap ${SRS_HOME}/mgmt/bootstrap
mkdir -p ${SRS_HOME}/mgmt && cp ${SOURCE}/mgmt/bootstrap ${SRS_HOME}/mgmt/bootstrap
if [[ $? -ne 0 ]]; then echo "Copy srs-cloud failed"; exit 1; fi

########################################################################################################################
Expand Down Expand Up @@ -139,7 +138,7 @@ fi
# Create srs-cloud service, and the credential file.
# Remark: Never start the service, because the IP will change for new machine created.
cp -f ${SRS_HOME}/usr/lib/systemd/system/srs-cloud.service /usr/lib/systemd/system/srs-cloud.service &&
systemctl enable srs-cloud
systemctl daemon-reload && systemctl enable srs-cloud
if [[ $? -ne 0 ]]; then echo "Install srs-cloud failed"; exit 1; fi

########################################################################################################################
Expand Down
76 changes: 76 additions & 0 deletions scripts/tools/tencent-cloud/create-verify-cvm.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#coding: utf-8
import dotenv, os, time, sys, tools, argparse

parser = argparse.ArgumentParser(description="TencentCloud")
parser.add_argument("--image", type=str, required=False, help="The CVM image id")

args = parser.parse_args()

if os.path.exists(f'{os.getenv("HOME")}/.lighthouse/.env'):
dotenv.load_dotenv(dotenv.find_dotenv(filename=f'{os.getenv("HOME")}/.lighthouse/.env'))
else:
dotenv.load_dotenv(dotenv.find_dotenv())

if os.getenv("VM_IMAGE") is not None and args.image is None:
args.image = os.getenv("VM_IMAGE")
if args.image == None:
raise Exception("Please set --image")

if os.getenv("SECRET_ID") == None:
print("Please set SECRET_ID in .env or ~/.lighthouse/.env file")
exit(1)
if os.getenv("SECRET_KEY") == None:
print("Please set SECRET_KEY in .env or ~/.lighthouse/.env file")
exit(1)
if os.getenv("VM_TOKEN") == None:
print("Please set VM_TOKEN in .env or ~/.lighthouse/.env file")
exit(1)

region = "ap-beijing"
image_id = args.image
print(f"Select image: {image_id}, {region}")

instance_quotas = tools.get_zone_instance(region)['InstanceTypeQuotaSet']
if len(instance_quotas) == 0:
raise Exception("No instance type found")
instance_candidates = []
for v in instance_quotas:
if v['Cpu'] != 2 or v['Fpga'] != 0 or v['Gpu'] != 0 or v['Memory'] != 2 or v['Status'] != 'SELL':
continue
instance_candidates.append(v)
if len(instance_candidates) == 0:
raise Exception("No instance type found")
zone = instance_candidates[0]['Zone']
instance_type = instance_candidates[0]['InstanceType']
vm_token = os.getenv("VM_TOKEN")
print(f"Create CVM in region={region}, zone={zone}, instance={instance_type}, image={image_id}, vm_token={vm_token}")

disk = 10
network = 30
instance_ids = tools.create_instance(region, zone, instance_type, image_id, disk, network, vm_token)['InstanceIdSet']
print(f"Created instances {instance_ids}")
if len(instance_ids) < 1:
raise Exception("Create instance failed")

instance_id = instance_ids[0]
print(f"CVM instance={instance_id}, region={region}, zone={zone}, instance={instance_type}, image={image_id}, disk={disk}GB, network={network}Mbps")

while True:
instance_states = tools.query_instance_status(region, instance_id)['InstanceStatusSet']
if len(instance_states) != 1:
raise Exception(f"Instance {instance_id} status not found")
instance_state = instance_states[0]
if instance_state['InstanceState'] == 'RUNNING':
print(f"Instance {instance_id} is running")
break
print(f"Instance {instance_id} is {instance_state['InstanceState']}, wait 3 seconds")
time.sleep(3)

instance_details = tools.query_instance_detail(region, instance_id)['InstanceSet']
if len(instance_details) != 1:
raise Exception(f"Instance {instance_id} detail not found")
instance_detail = instance_details[0]
print(f"Instance {instance_id}, public ip={instance_detail['PublicIpAddresses'][0]}, private ip={instance_detail['PrivateIpAddresses'][0]}")

# print the instance id to stderr.
print(instance_id, file=sys.stderr)
24 changes: 12 additions & 12 deletions scripts/tools/tencent-cloud/remove-image.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import dotenv, os, tools, argparse

parser = argparse.ArgumentParser(description="TencentCloud")
parser.add_argument("--instance", type=str, required=False, help="The CVM instance id")
parser.add_argument("--image", type=str, required=False, help="The CVM image id")

args = parser.parse_args()

Expand All @@ -11,10 +11,10 @@
else:
dotenv.load_dotenv(dotenv.find_dotenv())

if os.getenv("VM_IMAGE") is not None and args.instance is None:
args.instance = os.getenv("VM_IMAGE")
if args.instance == None:
raise Exception("Please set --instance")
if os.getenv("VM_IMAGE") is not None and args.image is None:
args.image = os.getenv("VM_IMAGE")
if args.image == None:
raise Exception("Please set --image")

if os.getenv("SECRET_ID") == None:
print("Please set SECRET_ID in .env or ~/.lighthouse/.env file")
Expand All @@ -24,14 +24,14 @@
exit(1)

region = "ap-beijing"
instance_id = args.instance
print(f"Remove CVM image id={instance_id}, region={region}")
image_id = args.image
print(f"Remove CVM image id={image_id}, region={region}")

shares = tools.get_image_share(region, instance_id)['SharePermissionSet']
shares = tools.get_image_share(region, image_id)['SharePermissionSet']
account_ids = [item['AccountId'] for item in shares]
if len(account_ids) > 0:
tools.cancel_image_share(region, instance_id, account_ids)
print(f"Image={instance_id}, Shares={len(shares)}, Accounts={account_ids}, Canceled")
tools.cancel_image_share(region, image_id, account_ids)
print(f"Image={image_id}, Shares={len(shares)}, Accounts={account_ids}, Canceled")

tools.delete_image_and_snapshot(region, instance_id)
print(f"Image {instance_id} deleted")
tools.delete_image_and_snapshot(region, image_id)
print(f"Image {image_id} deleted")
31 changes: 21 additions & 10 deletions scripts/tools/tencent-cloud/share-image.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#coding: utf-8
import dotenv, os, tools, argparse
import dotenv, os, tools, argparse, time

parser = argparse.ArgumentParser(description="TencentCloud")
parser.add_argument("--instance", type=str, required=False, help="The CVM instance id")
parser.add_argument("--image", type=str, required=False, help="The CVM image id")

args = parser.parse_args()

Expand All @@ -11,10 +11,10 @@
else:
dotenv.load_dotenv(dotenv.find_dotenv())

if os.getenv("VM_IMAGE") is not None and args.instance is None:
args.instance = os.getenv("VM_IMAGE")
if args.instance == None:
raise Exception("Please set --instance")
if os.getenv("VM_IMAGE") is not None and args.image is None:
args.image = os.getenv("VM_IMAGE")
if args.image == None:
raise Exception("Please set --image")

if os.getenv("SECRET_ID") == None:
print("Please set SECRET_ID in .env or ~/.lighthouse/.env file")
Expand All @@ -27,9 +27,20 @@
exit(1)

region = "ap-beijing"
instance_id = args.instance
image_id = args.image
account_id = os.getenv("LH_ACCOUNT")
print(f"Share image id={instance_id}, region={region} to account={account_id}")
print(f"Share image id={image_id}, region={region} to account={account_id}")

tools.share_image(region, instance_id, account_id)
print(f"Image {instance_id} shared to account {account_id}")
while True:
info = tools.query_image(region, image_id)['ImageSet']
if len(info) != 1:
raise Exception(f"Image {image_id} not found")

if info[0]['ImageState'] == 'NORMAL':
break

print(f"Image {image_id} state is {info[0]['ImageState']}, wait 5 seconds")
time.sleep(5)

tools.share_image(region, image_id, account_id)
print(f"Image {image_id} shared to account {account_id}")
18 changes: 18 additions & 0 deletions scripts/tools/tencent-cloud/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -373,3 +373,21 @@ def share_image(region, image_id, account_id):

resp = client.ModifyImageSharePermission(req)
return json.loads(resp.to_json_string())

def query_image(region, image_id):
cred = credential.Credential(os.getenv("SECRET_ID"), os.getenv("SECRET_KEY"))
httpProfile = HttpProfile()
httpProfile.endpoint = "cvm.tencentcloudapi.com"

clientProfile = ClientProfile()
clientProfile.httpProfile = httpProfile
client = cvm_client.CvmClient(cred, region, clientProfile)

req = cvm_models.DescribeImagesRequest()
params = {
"ImageIds": [image_id]
}
req.from_json_string(json.dumps(params))

resp = client.DescribeImages(req)
return json.loads(resp.to_json_string())

0 comments on commit e3d99e8

Please sign in to comment.