HDDS-3995. Fix s3g met NPE exception while write file by multiPartUpload

hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config

@@ -112,4 +112,4 @@ JSVC_HOME=/usr/bin
 SLEEP_SECONDS=5
 KERBEROS_ENABLED=true
 
-no_proxy=om,scm,recon,s3g,kdc,localhost,127.0.0.1
+no_proxy=om,scm,recon,s3g,recon,kdc,localhost,127.0.0.1

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -67,9 +67,14 @@ wait_for_safemode_exit(){
 
   #Reset the timer
   SECONDS=0
+  timeout_threshold=240
+  if [[ "${SECURITY_ENABLED}" == 'true' ]]; then
+     timeout_threshold=600
+  fi
+
+  #Don't give it up until timeout
+  while [[ $SECONDS -lt $timeout_threshold ]]; do
 
-  #Don't give it up until 240 seconds
-  while [[ $SECONDS -lt 240 ]]; do
 
      #This line checks the safemode status in scm
      local command="${OZONE_SAFEMODE_STATUS_COMMAND}"
@@ -141,6 +146,8 @@ execute_robot_test(){
     && docker-compose exec -T "$CONTAINER" robot -v OM_SERVICE_ID:"${OM_SERVICE_ID}" -v SECURITY_ENABLED:"${SECURITY_ENABLED}" -v OM_HA_PARAM:"${OM_HA_PARAM}" -v KEY_NAME:"${OZONE_BUCKET_KEY_NAME}" ${ARGUMENTS[@]} --log NONE --report NONE "${OZONE_ROBOT_OPTS[@]}" --output "$OUTPUT_PATH" "$SMOKETEST_DIR_INSIDE/$TEST"
   local -i rc=$?
 
+
+  export COLUMNS=500 #defaul width of docker-compose terminal output is too narrow(80)
   FULL_CONTAINER_NAME=$(docker-compose ps | grep "_${CONTAINER}_" | head -n 1 | awk '{print $1}')
   docker cp "$FULL_CONTAINER_NAME:$OUTPUT_PATH" "$RESULT_DIR/"
 

hadoop-ozone/dist/src/main/compose/upgrade/test.sh

@@ -26,6 +26,8 @@ export COMPOSE_DIR
 export OZONE_VOLUME
 
 mkdir -p "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm}
+mkdir -p "${OZONE_VOLUME}/debug"
+chmod -R 777 "${OZONE_VOLUME}"/{dn1,dn2,dn3,om,recon,s3g,scm,debug}
 
 if [[ -n "${OZONE_VOLUME_OWNER}" ]]; then
   current_user=$(whoami)

hadoop-ozone/recon/pom.xml

@@ -110,6 +110,7 @@
             </goals>
             <configuration>
               <arguments>pnpm config set store-dir ~/.pnpm-store</arguments>
+              <npmInheritsProxyConfigFromMaven>false</npmInheritsProxyConfigFromMaven>
             </configuration>
           </execution>
           <execution>
@@ -119,6 +120,7 @@
             </goals>
             <configuration>
               <arguments>pnpm install</arguments>
+              <npmInheritsProxyConfigFromMaven>false</npmInheritsProxyConfigFromMaven>
             </configuration>
           </execution>
           <execution>
@@ -128,6 +130,7 @@
             </goals>
             <configuration>
               <arguments>pnpm run build</arguments>
+              <npmInheritsProxyConfigFromMaven>false</npmInheritsProxyConfigFromMaven>
             </configuration>
           </execution>
         </executions>

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java

@@ -598,7 +598,11 @@ private Response createMultipartKey(String bucket, String key, long length,
           IOUtils.copy(body, ozoneOutputStream);
         }
       } finally {
-        IOUtils.closeQuietly(ozoneOutputStream);
+        if (ozoneOutputStream != null) {
+          ozoneOutputStream.close();          
+        } else {
+          throw S3ErrorTable.newError(NO_SUCH_UPLOAD, uploadID);
+        }
       }
 
       OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo =
@@ -611,7 +615,6 @@ private Response createMultipartKey(String bucket, String key, long length,
         return Response.ok().header("ETag",
             eTag).build();
       }
-
     } catch (OMException ex) {
       if (ex.getResult() == ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR) {
         throw S3ErrorTable.newError(NO_SUCH_UPLOAD,

tencent/python-client/OzoneS3.py

@@ -0,0 +1,89 @@
+import boto3
+from botocore.config import Config
+from botocore.exceptions import ClientError
+import xml.etree.ElementTree as ET
+from itertools import izip_longest
+from random import randrange
+
+# Ozone doesn't care access key and secret key in non-SafeMode
+AWS_ACCESS_KEY_ID = '12345678'
+AWS_SECRET_ACCESS_KEY = '12345678'
+
+# Ozone S3 gateway endpoints
+root = ET.parse('ozone_idex_config.xml').getroot()
+endpoints = root.findall('property/entry')
+SERVER_LIST = []
+for endpoint in endpoints:
+    SERVER_LIST.append(endpoint.get('ozone.s3g.endpoint'))
+#endpoint = 'http://localhost:9878'
+
+# Ozone S3 bucket name
+bucket_name = 'idex_bucket'
+
+def round_robin(cur = [0]):
+    length = len(SERVER_LIST)
+    ret = SERVER_LIST[cur[0] % length]
+    cur[0] = (cur[0] + 1) % length
+    return ret
+
+def s3_resource():
+    session = boto3.session.Session(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+    endpoint = round_robin([randrange(len(SERVER_LIST))])
+    s3 = session.resource(service_name='s3', endpoint_url=endpoint)
+    print("current endpoint: " + endpoint)
+    return s3
+
+class OzoneS3:
+    @staticmethod
+    def upload_file(file_path):
+        try:
+            s3_resource().Bucket(bucket_name).upload_file(file_path, file_path)
+        except Exception as e:
+            print("retry one time. exception: " + str(e))
+            s3_resource().Bucket(bucket_name).upload_file(file_path, file_path)
+
+
+    @staticmethod
+    def upload_stream(key, byte_stream):
+        try:
+            s3_resource().Object(bucket_name, key).put(Body=byte_stream)
+        except Exception as e:
+            s3_resource().Object(bucket_name, key).put(Body=byte_stream)
+
+    @staticmethod
+    def download_file(key, output_file_path):
+        try:
+            s3_resource().Bucket(bucket_name).download_file(key, output_file_path)
+        except Exception as e:
+            s3_resource().Bucket(bucket_name).download_file(key, output_file_path)
+
+    @staticmethod
+    def download_stream(key):
+        try:
+            obj = s3_resource().Object(bucket_name, key)
+            return obj.get()['Body'].read()
+        except Exception as e:
+            obj = s3_resource().Object(bucket_name, key)
+            return obj.get()['Body'].read()
+
+    @staticmethod
+    def iter_n(iterable, n, fillvalue=None):
+        args = [iter(iterable)] * n
+        return izip_longest(*args, fillvalue=fillvalue)
+
+    @staticmethod
+    def download_stream_iter(key):
+        try:
+            obj = s3_resource().Object(bucket_name, key)
+            return obj.get()['Body']
+        except Exception as e:
+            obj = s3_resource().Object(bucket_name, key)
+            return obj.get()['Body']
+
+    @staticmethod
+    def delete(key):
+        try:
+            s3_resource().Object(bucket_name, key).delete()
+        except Exception as e:
+            s3_resource().Object(bucket_name, key).delete()
+

tencent/python-client/README.md

@@ -0,0 +1,6 @@
+Python interface for ozone thru s3 gateway.
+
+In order to use this OzoneS3.py, make sure python3 is installed in your env. Then put s3 gateway endpoint in ozone_client_confg.xml.
+If you have multiple s3 gateway endpoint, put separate entries in ozone_client_config.xml. OzoneS3.py will randomly pick one.
+
+Bucket is hardcoded in OzoneS3.py. In terms of s3 gateway of Ozone, please refer to https://hadoop.apache.org/ozone/docs/0.4.0-alpha/s3.html .

tencent/python-client/ozone_client_config.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ozone_config>
+   <property>
+     <entry ozone.s3g.endpoint='http://localhost:9878'/>
+     <entry ozone.s3g.endpoint='http://127.0.0.1:9878'/>
+   </property>
+</ozone_config>