From 70d25828f870bbc23ce04f134d89b37ed6d36ac5 Mon Sep 17 00:00:00 2001 From: Vincent Rioux Date: Thu, 3 Aug 2023 11:45:51 -0400 Subject: [PATCH 1/3] Update to PHP8.1, Amazon Linux 2023, fix opcache issue --- templates/04-web.yaml | 115 +++++++++++++++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 23 deletions(-) diff --git a/templates/04-web.yaml b/templates/04-web.yaml index 3af16dc..d603bce 100644 --- a/templates/04-web.yaml +++ b/templates/04-web.yaml @@ -142,14 +142,18 @@ Parameters: Type: AWS::EC2::SecurityGroup::Id LatestAmiId : Type : AWS::SSM::Parameter::Value - Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 LatestArmAmiId : Type : AWS::SSM::Parameter::Value - Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-arm64-gp2 + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 CodeArtifactS3BucketArn: Type: "String" Description: Code Artifact S3 Bucket Arn - + ProjectName: + AllowedPattern: ^([a-zA-Z0-9]*)$ + Default: App + Description: Main stack name + Type: String Conditions: NumberOfSubnets1: @@ -335,9 +339,6 @@ Resources: - start_webserver - add_crontab install_logs: - packages: - yum: - awslogs: [] files: /etc/awslogs/awslogs.conf: content: !Sub | @@ -394,12 +395,8 @@ Resources: commands: 01_create_state_directory: command: mkdir -p /var/awslogs/state - services: - sysvinit: - awslogsd: - enabled: 'true' - ensureRunning: 'true' - files: /etc/awslogs/awslogs.conf + 02_install_cloudwatch_agent: + command: dnf install -y amazon-cloudwatch-agent install_aws_ini: commands: install_aws_ini: @@ -407,14 +404,12 @@ Resources: cwd: /tmp ignoreErrors: true install_codedeploy: - packages: - yum: - ruby: [] files: /tmp/install_codedeploy.sh: content: !Sub | #!/bin/bash -xe + dnf install -y ruby cd /home/ec2-user wget https://aws-codedeploy-${AWS::Region}.s3.${AWS::Region}.amazonaws.com/latest/install chmod +x ./install @@ -438,13 +433,34 @@ Resources: /tmp/create_site_conf.sh: content: !Sub | #!/bin/bash -xe - amazon-linux-extras install -y php8.0 - amazon-linux-extras enable php8.0 + dnf install -y php8.1 php-gd php-soap php-intl php-mbstring php-xml php-opcache php-fpm php-pgsql php-mysqlnd + + # Enable crontab on Amazon Linux 2023 + dnf install -y cronie + systemctl enable crond + systemctl start crond - yum install -y httpd gcc-c++ - yum install -y php-gd php-soap php-intl php-mbstring php-xml php-zip php-opcache php-sodium php-fpm + # Below to be able to compile zip.so for the PHP Zip library that's not in the available packages... + dnf install -y php8.1-devel php-pear libzip libzip-devel + pecl install zip + echo "extension=zip.so;" > /etc/php.d/50-zip.ini + # Install Sodium + dnf install -y gcc + wget https://download.libsodium.org/libsodium/releases/LATEST.tar.gz + tar -xvzf LATEST.tar.gz + cd libsodium-stable + ./configure + make + make install + pecl install -f libsodium + echo "extension=sodium.so;" > /etc/php.d/50-sodium.ini + + # Adjust base php.ini sed -i 's/memory_limit =.*/memory_limit = 4096M/' /etc/php.ini + sed -i 's/;max_input_vars.*/max_input_vars = 5000/' /etc/php.ini + + # Create Apache config if [ ! -f /etc/httpd/conf.d/moodle.conf ]; then touch /etc/httpd/conf.d/moodle.conf echo 'ServerName 127.0.0.1:80' >> /etc/httpd/conf.d/moodle.conf @@ -456,6 +472,62 @@ Resources: echo '' >> /etc/httpd/conf.d/moodle.conf fi + # Create hidden opcache directory locally & change owner to apache + if [ ! -d /var/www/.opcache ]; then + mkdir -p /var/www/.opcache + fi + # Ensure opcache is enabled and add settings recomended by moodle at https://docs.moodle.org/34/en/OPcache + sed -i 's/;opcache.file_cache=.*/opcache.file_cache=\/var\/www\/.opcache/' /etc/php.d/10-opcache.ini + sed -i 's/opcache.memory_consumption=.*/opcache.memory_consumption=512/' /etc/php.d/10-opcache.ini + sed -i 's/opcache.max_accelerated_files=.*/opcache.max_accelerated_files=8000/' /etc/php.d/10-opcache.ini + sed -i 's/;opcache.revalidate_freq=.*/opcache.revalidate_freq=300/' /etc/php.d/10-opcache.ini + sed -i 's/;opcache.use_cwd=.*/opcache.use_cwd=1/' /etc/php.d/10-opcache.ini + sed -i 's/;opcache.validate_timestamps=.*/opcache.validate_timestamps=1/' /etc/php.d/10-opcache.ini + sed -i 's/;opcache.save_comments=.*/opcache.save_comments=1/' /etc/php.d/10-opcache.ini + sed -i 's/;opcache.enable_file_override=.*/opcache.enable_file_override=60/' /etc/php.d/10-opcache.ini + sed -i 's/;opcache.file_cache_only=.*/opcache.file_cache_only=1/' /etc/php.d/10-opcache.ini + + # Install ElastiCache client + if [ $(uname -a | grep -c x86_64) == "1" ]; then + echo "downloading x86 client for ElastiCache" + wget -P /tmp/ https://elasticache-downloads.s3.amazonaws.com/ClusterClient/PHP-8.1/latest-64bit-X86-openssl3 + tar -xf '/tmp/latest-64bit-X86-openssl3' + else + echo "downloading ARM-64 client for ElastiCache" + wget -P /tmp/ https://elasticache-downloads.s3.amazonaws.com/ClusterClient/PHP-8.2/latest-64bit-arm-X86-openssl3 + tar -xf '/tmp/latest-64bit-arm-X86-openssl3' + fi + mv amazon-elasticache-cluster-client.so /usr/lib64/php/modules/ + echo 'extension=amazon-elasticache-cluster-client.so;' > /etc/php.d/50-elasticache.ini + + # Mount EFS + availabilityzone=$(ec2-metadata -z | awk '{print $2}' | sed 's/(.)//') + region=$(ec2-metadata -z | awk '{print $2}' | sed 's/[a-z]$//') + if grep -qs '/var/www/moodle/data ' /proc/mounts; then + echo "/var/www/moodle/data is mounted." + else + export EnvElasticFileSystem=$(aws ssm get-parameters --region $region --names /Moodle/${ProjectName}/SharedFile/ElasticFileSystem --query Parameters[0].Value) + export EnvElasticFileSystem=`echo $EnvElasticFileSystem | sed -e 's/^"//' -e 's/"$//'` + + sudo mkdir -p /$EnvElasticFileSystem + sudo mountpoint -q /$EnvElasticFileSystem || sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $EnvElasticFileSystem.efs.${AWS::Region}.amazonaws.com:/ /$EnvElasticFileSystem + + #Create directories for Moodle + sudo mkdir -p /$EnvElasticFileSystem/data + sudo mkdir -p /$EnvElasticFileSystem/cache + sudo mkdir -p /$EnvElasticFileSystem/temp + + chown apache:apache /$EnvElasticFileSystem/data/ + chown apache:apache /$EnvElasticFileSystem/cache/ + chown apache:apache /$EnvElasticFileSystem/temp/ + + sudo umount -f /$EnvElasticFileSystem + + mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $EnvElasticFileSystem.efs.${AWS::Region}.amazonaws.com:/data /var/www/moodle/data + #mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $EnvElasticFileSystem.efs.${AWS::Region}.amazonaws.com:/cache /var/www/moodle/cache + #mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $EnvElasticFileSystem.efs.${AWS::Region}.amazonaws.com:/temp /var/www/moodle/temp + fi + cp /tmp/status.txt /var/www/moodle/html/status.txt mode: 000500 owner: root @@ -476,7 +548,7 @@ Resources: /tmp/add_crontab.sh: content: !Sub | #!/bin/sh -xe - echo "* * * * * apache /usr/bin/php /var/www/moodle/html/admin/cli/cron.php" > /etc/cron.d/moodle + echo "* * * * * apache /usr/bin/php /var/www/moodle/html/admin/cli/cron.php" >> /etc/cron.d/moodle mode: 000500 owner: root group: root @@ -508,9 +580,6 @@ Resources: #!/bin/bash -xe sudo systemctl enable amazon-ssm-agent sudo systemctl start amazon-ssm-agent - sudo systemctl status amazon-ssm-agent - - yum update -y #Create directory structure mkdir -p /var/www/moodle/html From bf977a0768ddd4ee40fe110e6a0f582f11959abc Mon Sep 17 00:00:00 2001 From: Vincent Rioux Date: Thu, 3 Aug 2023 11:46:37 -0400 Subject: [PATCH 2/3] Update to Amazon Linux 2023 --- templates/03-pipelinehelper.yaml | 160 +++---------------------------- 1 file changed, 11 insertions(+), 149 deletions(-) diff --git a/templates/03-pipelinehelper.yaml b/templates/03-pipelinehelper.yaml index beab4b5..39ee2c4 100644 --- a/templates/03-pipelinehelper.yaml +++ b/templates/03-pipelinehelper.yaml @@ -107,10 +107,10 @@ Parameters: LatestAmiId: Type : AWS::SSM::Parameter::Value - Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 LatestArmAmiId : Type : AWS::SSM::Parameter::Value - Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-arm64-gp2 + Default: /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 MoodleLocale: Description: "The main language of the Moodle site, during initial configuration." @@ -167,6 +167,7 @@ Conditions: - !Equals ["r6",!Select [0, !Split [ "g.", !Ref InstanceType]]] Resources: +########################### TODO put this in the codepipeline template MoodleRepo: Type: AWS::CodeCommit::Repository Properties: @@ -265,9 +266,6 @@ Resources: moodle_git_config: - moodle-git-config moodle-git-config: - packages: - yum: - git: [] files: /tmp/appspec.yml: content: !Sub | @@ -300,7 +298,6 @@ Resources: #!/bin/bash sudo systemctl start php-fpm sudo systemctl start httpd - mode: '000755' owner: root group: root @@ -457,8 +454,8 @@ Resources: chown -R apache:apache /var/www/moodle/temp chown -R apache:apache /var/www/moodle/local - availabilityzone=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) - region=${!availabilityzone:0:-1} + availabilityzone=$(ec2-metadata -z | awk '{print $2}' | sed 's/(.)//') + region=$(ec2-metadata -z | awk '{print $2}' | sed 's/[a-z]$//') export EnvDatabaseType=$(aws ssm get-parameters --region $region --names /Moodle/${ProjectName}/DB/Type --query Parameters[0].Value) export EnvDatabaseType=`echo $EnvDatabaseType | sed -e 's/^"//' -e 's/"$//'` @@ -494,7 +491,7 @@ Resources: export EnvElastiCacheEngine=$(aws ssm get-parameters --region $region --names /Moodle/${ProjectName}/Cache/session/Engine --query Parameters[0].Value) export EnvElastiCacheEngine=`echo $EnvElastiCacheEngine | sed -e 's/^"//' -e 's/"$//'` - #setting up elasticache dependencies for cache + # Setting up ElastiCache dependencies for cache if [ "$EnvIsMoodleSetupCompleted" != "No" ] && [ "$EnvElastiCacheClusterEndpointAddress" != "null" -a "$EnvElastiCacheClusterEndpointAddress" != "" ]; then sed -i "s/\$SessionEndpoint = .*/\$SessionEndpoint = '"$EnvElastiCacheClusterEndpointAddress"';/" /var/www/moodle/html/config.php @@ -504,154 +501,20 @@ Resources: #update Moodle source to use DYNAMIC_CLIENT_MODE so Moodle can detect changes to the elasticache cluster membership sed -i '/\$this->options\[Memcached::OPT_BUFFER_WRITES\] = \$bufferwrites;/a \ \ \ \ \ \ \ \ $this->options[Memcached::OPT_CLIENT_MODE] = Memcached::DYNAMIC_CLIENT_MODE;' /var/www/moodle/html/cache/stores/memcached/lib.php fi - - else sed -i "s/\$SessionEndpoint = .*/\$SessionEndpoint = '';/" /var/www/moodle/html/config.php fi - - sudo systemctl restart php-fpm mode: 000500 owner: root group: root - /tmp/before_install.sh: content: !Sub | #!/bin/bash -xe - - cd /opt/codedeploy-agent/deployment-root/$DEPLOYMENT_GROUP_ID/$DEPLOYMENT_ID/deployment-archive/.pipeline/ - - availabilityzone=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) - region=${!availabilityzone:0:-1} - - export EnvDatabaseType=$(aws ssm get-parameters --region $region --names /Moodle/${ProjectName}/DB/Type --query Parameters[0].Value) - export EnvDatabaseType=`echo $EnvDatabaseType | sed -e 's/^"//' -e 's/"$//'` - - if [ "$EnvDatabaseType" == "MySQL" ]; then - #Installing and configuring MYSQL libs. - sh install_mysql_dependencies.sh - else - #Installing and configuring PGSQL libs. - sh install_pgsql_dependencies.sh - fi - - #increasing PHP max_input_vars to 5000 - sed -i 's/; max_input_vars.*/max_input_vars = 5000/' /etc/php.ini - - #Configuring OPCache - sh configure_opcache.sh - - #configuring Cache clients - sh install_cacheclient.sh - - # Setting up EFS shared file storage - sh setup_efs.sh - mode: 000500 - owner: root - group: root - /tmp/setup_efs.sh: - content: - !Sub | - #!/bin/bash -xe - - availabilityzone=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) - region=${!availabilityzone:0:-1} - - #Mount shared storage - if grep -qs '/var/www/moodle/data ' /proc/mounts; then - echo "/var/www/moodle/data is mounted." - else - export EnvElasticFileSystem=$(aws ssm get-parameters --region $region --names /Moodle/${ProjectName}/SharedFile/ElasticFileSystem --query Parameters[0].Value) - export EnvElasticFileSystem=`echo $EnvElasticFileSystem | sed -e 's/^"//' -e 's/"$//'` - - sudo mkdir -p /$EnvElasticFileSystem - sudo mountpoint -q /$EnvElasticFileSystem || sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $EnvElasticFileSystem.efs.${AWS::Region}.amazonaws.com:/ /$EnvElasticFileSystem - - #Create directories for Moodle - sudo mkdir -p /$EnvElasticFileSystem/data - sudo mkdir -p /$EnvElasticFileSystem/cache - sudo mkdir -p /$EnvElasticFileSystem/temp - - chown apache:apache /$EnvElasticFileSystem/data/ - chown apache:apache /$EnvElasticFileSystem/cache/ - chown apache:apache /$EnvElasticFileSystem/temp/ - - sudo umount -f /$EnvElasticFileSystem - - mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $EnvElasticFileSystem.efs.${AWS::Region}.amazonaws.com:/data /var/www/moodle/data - #mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $EnvElasticFileSystem.efs.${AWS::Region}.amazonaws.com:/cache /var/www/moodle/cache - #mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 $EnvElasticFileSystem.efs.${AWS::Region}.amazonaws.com:/temp /var/www/moodle/temp - fi - mode: 000500 - owner: root - group: root - /tmp/install_pgsql_dependencies.sh: - content: - !Sub | - #!/bin/bash -xe - amazon-linux-extras install -y postgresql13 - yum install -y php-pgsql + echo "Nothing to do" mode: 000500 owner: root group: root - /tmp/install_mysql_dependencies.sh: - content: - !Sub | - #!/bin/bash -xe - - amazon-linux-extras install -y mariadb10.5 - yum install -y php-mysqlnd - mode: 000500 - owner: root - group: root - /tmp/configure_opcache.sh: - content: - !Sub | - #!/bin/bash -xe - # create hidden opcache directory locally & change owner to apache - if [ ! -d /var/www/.opcache ]; then - mkdir -p /var/www/.opcache - fi - #Ensure opcache is enabled and add settings recomended by moodle at https://docs.moodle.org/34/en/OPcache - sed -i 's/;opcache.file_cache=.*/opcache.file_cache=\/var\/www\/.opcache/' /etc/php.d/10-opcache.ini - sed -i 's/opcache.memory_consumption=.*/opcache.memory_consumption=512/' /etc/php.d/10-opcache.ini - sed -i 's/opcache.max_accelerated_files=.*/opcache.max_accelerated_files=8000/' /etc/php.d/10-opcache.ini - sed -i 's/;opcache.revalidate_freq=.*/opcache.revalidate_freq=300/' /etc/php.d/10-opcache.ini - sed -i 's/;opcache.use_cwd=.*/opcache.use_cwd=1/' /etc/php.d/10-opcache.ini - sed -i 's/;opcache.validate_timestamps=.*/opcache.validate_timestamps=1/' /etc/php.d/10-opcache.ini - sed -i 's/;opcache.save_comments=.*/opcache.save_comments=1/' /etc/php.d/10-opcache.ini - sed -i 's/;opcache.enable_file_override=.*/opcache.enable_file_override=60/' /etc/php.d/10-opcache.ini - mode: 000500 - owner: root - group: root - /tmp/install_cacheclient.sh: - content: - !Sub | - #!/bin/bash -xe - - #Install memcached and then remove it. Memcached is not actually needed. We install amazon-elasticache-cluster-client.so instead. However Moodle does not detect memcached is installed. Therefore, this tricks Moodle into thinking it is installed. - sudo yum install -y php-pecl-memcached - sudo yum remove -y php-pecl-memcached - sudo yum install -y php-redis - sudo yum install -y openssl11 - - if [ $(uname -a | grep -c x86_64) == "1" ]; then - echo "downloading x86 client for ElastiCache" - wget -P /tmp/ https://elasticache-downloads.s3.amazonaws.com/ClusterClient/PHP-8.0/latest-64bit-X86-openssl1.1 - tar -xf '/tmp/latest-64bit-X86-openssl1.1' - else - echo "downloading ARM-64 client for ElastiCache" - wget -P /tmp/ https://elasticache-downloads.s3.amazonaws.com/ClusterClient/PHP-8.0/latest-64bit-arm-openssl1.1 - tar -xf '/tmp/latest-64bit-arm-openssl1.1' - fi - - cp 'amazon-elasticache-cluster-client.so' /usr/lib64/php/modules/ - echo 'extension=amazon-elasticache-cluster-client.so;' > /etc/php.d/50-elasticache.ini - mode: 000500 - owner: root - group: root - /tmp/moodle-git-config.sh: content: !Sub | #!/bin/bash -x @@ -707,11 +570,11 @@ Resources: git push -u origin main # get instance id - instance_id=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) + instance_id=$(ec2-metadata -i | awk '{print $2}' | sed 's/(.)//') # get region from instance meta-data - availabilityzone=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone) - region=${!availabilityzone:0:-1} + availabilityzone=$(ec2-metadata -z | awk '{print $2}' | sed 's/(.)//') + region=$(ec2-metadata -z | awk '{print $2}' | sed 's/[a-z]$//') # wait for Moodle setup to be completed echo "Start checking whether Moodle setup completed or not" @@ -756,7 +619,6 @@ Resources: command: ./moodle-git-config.sh cwd: /tmp ignoreErrors: false - Properties: LaunchTemplateData: BlockDeviceMappings: @@ -778,7 +640,7 @@ Resources: #!/bin/bash -xe sudo systemctl enable amazon-ssm-agent sudo systemctl start amazon-ssm-agent - sudo systemctl status amazon-ssm-agent + dnf install -y git /opt/aws/bin/cfn-init --configsets moodle_git_config --verbose --stack ${AWS::StackName} --resource PipelineHelperLaunchTemplate --region ${AWS::Region} /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource PipelineHelperASGroup --region ${AWS::Region} From 763b92b986c8872f371edb429b8c41b56daf296b Mon Sep 17 00:00:00 2001 From: Vincent Rioux Date: Thu, 3 Aug 2023 11:47:07 -0400 Subject: [PATCH 3/3] Update to PHP8.1, Amazon Linux 2023 --- templates/00-main.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/00-main.yaml b/templates/00-main.yaml index b854f34..7b85f8a 100644 --- a/templates/00-main.yaml +++ b/templates/00-main.yaml @@ -163,7 +163,7 @@ Parameters: DeploymentLocation: Description: Location to deploy from (S3 URL), Keep it as is unless you created your own S3 bucket Type: String - Default: https://s3.amazonaws.com/aws-refarch/moodle/latest/templates + Default: https://s3.amazonaws.com/aws-refarch/moodle/al2023/templates BastionInstanceType: AllowedValues: - t3.nano @@ -1013,6 +1013,8 @@ Resources: WebSecurityGroup: !GetAtt [ securitygroups, Outputs.WebSecurityGroup ] CodeArtifactS3BucketArn: !GetAtt [ pipelineHelper, Outputs.CodeArtifactS3BucketArn] + ProjectName: + !Sub '${AWS::StackName}' TemplateURL: !Sub '${DeploymentLocation}/04-web.yaml' codePipeline: