Merge branch 'main' into runtime_filter_query_log

tests/suites/1_stateful/05_formats/05_00_01_load_unload_all.result

@@ -1,15 +1,9 @@
 ---CSV
 "a""b",1.0,"['a""b']","{""k"":""v""}","2044-05-06 10:25:02.868894",10.01,"{'k1':10,'k2':20}","('a',5)"
 \N,2.0,"['a''b']","[1]","2044-05-06 10:25:02.868894",-10.01,"{}","('b',10)"
-SUCCESS
-null
 ---TSV
 a"b	1.0	['a"b']	{"k":"v"}	2044-05-06 10:25:02.868894	10.01	{'k1':10,'k2':20}	('a',5)
 \N	2.0	['a''b']	[1]	2044-05-06 10:25:02.868894	-10.01	{}	('b',10)
-SUCCESS
-null
 ---NDJSON
 {"a":"a\"b","b":1.0,"c":["a\"b"],"d":{"k":"v"},"e":"2044-05-06 10:25:02.868894","f":10.01,"g":{"k1":10,"k2":20},"h":{"1":"a","2":5}}
 {"a":null,"b":2.0,"c":["a'b"],"d":[1],"e":"2044-05-06 10:25:02.868894","f":-10.01,"g":{},"h":{"1":"b","2":10}}
-SUCCESS
-null

tests/suites/1_stateful/05_formats/05_00_01_load_unload_all.sh

@@ -27,25 +27,28 @@ insert_data() {
 
 test_format() {
 	echo "---${1}"
+	rm -rf /tmp/test_load_unload
+	mkdir /tmp/test_load_unload
+	echo "drop stage if exists s1" | $BENDSQL_CLIENT_CONNECT
+	echo "create stage s1 url='fs:///tmp/test_load_unload/'" | $BENDSQL_CLIENT_CONNECT
 	echo "truncate table test_load_unload" | $BENDSQL_CLIENT_CONNECT
-	insert_data
-	rm -f /tmp/test_load_unload2.txt /tmp/test_load_unload.txt
 
-	curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" \
-	-d "select * from test_load_unload FORMAT ${1}" > /tmp/test_load_unload.txt
+	# insert
+	insert_data
 
-	cat /tmp/test_load_unload.txt
+	# unload1
+	echo "copy into @s1/unload1/ from test_load_unload file_format=(type=${1})" | $BENDSQL_CLIENT_CONNECT
+	mv `ls /tmp/test_load_unload/unload1/*` /tmp/test_load_unload/unload1.txt
+	cat /tmp/test_load_unload/unload1.txt
 
+	# load unload1
 	echo "truncate table test_load_unload" | $BENDSQL_CLIENT_CONNECT
+	echo "copy into test_load_unload from @s1/unload1.txt file_format=(type=${1}) force=true" | $BENDSQL_CLIENT_CONNECT
 
-	curl -sH "insert_sql:insert into test_load_unload file_format = (type = ${1})" \
-	-F "upload=@/tmp/test_load_unload.txt" \
-	-u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" | jq -r '.state, .error'
-
-	curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" \
-	-d "select * from test_load_unload FORMAT ${1}" > /tmp/test_load_unload2.txt
-
-	diff /tmp/test_load_unload2.txt /tmp/test_load_unload.txt
+	# unload2
+	echo "copy into @s1/unload2/ from test_load_unload file_format=(type=${1})" | $BENDSQL_CLIENT_CONNECT
+  mv `ls /tmp/test_load_unload/unload2/*` /tmp/test_load_unload/unload2.txt
+  diff /tmp/test_load_unload/unload1.txt /tmp/test_load_unload/unload2.txt
 }
 
 test_format "CSV"

tests/suites/1_stateful/05_formats/05_05_parquet/05_05_01_parquet_load_unload.sh

@@ -24,46 +24,41 @@ insert_data() {
 }
 
 test_format() {
+	rm -rf /tmp/test_load_unload
+	mkdir /tmp/test_load_unload
+	echo "drop stage if exists s1" | $BENDSQL_CLIENT_CONNECT
+	echo "create stage s1 url='fs:///tmp/test_load_unload/'" | $BENDSQL_CLIENT_CONNECT
+	echo "truncate table test_load_unload" | $BENDSQL_CLIENT_CONNECT
+
 	# insert data
 	insert_data
 
-	# unload clickhouse
-	curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" \
-	-d "select * from test_load_unload FORMAT ${1}" > /tmp/test_load_unload.parquet
+	# unload1
+	echo "copy into @s1/unload1/ from test_load_unload" | $BENDSQL_CLIENT_CONNECT
+	mv `ls /tmp/test_load_unload/unload1/*` /tmp/test_load_unload/unload1.parquet
 
 	echo "truncate table test_load_unload" | $BENDSQL_CLIENT_CONNECT
 
 	# load streaming
 	curl -sH "insert_sql:insert into test_load_unload file_format = (type = ${1})" \
-	-F "upload=@/tmp/test_load_unload.parquet" \
+	-F "upload=@/tmp/test_load_unload/unload1.parquet" \
 	-u root: -XPUT "http://localhost:${QUERY_HTTP_HANDLER_PORT}/v1/streaming_load" | grep -c "SUCCESS"
 
-	# unload clickhouse again
-	curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" \
-	-d "select * from test_load_unload FORMAT ${1}" > /tmp/test_load_unload2.parquet
+	# unload2
+	echo "copy into @s1/unload2/ from test_load_unload" | $BENDSQL_CLIENT_CONNECT
+	mv `ls /tmp/test_load_unload/unload2/*` /tmp/test_load_unload/unload2.parquet
 
 	echo "truncate table test_load_unload" | $BENDSQL_CLIENT_CONNECT
 
-	# copy into table
-	echo "copy into test_load_unload from 'fs:///tmp/test_load_unload.parquet' file_format = (type = ${1});" | $BENDSQL_CLIENT_CONNECT
-
-	# unload clickhouse again
-	curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" \
-	-d "select * from test_load_unload FORMAT ${1}" > /tmp/test_load_unload3.parquet
-
-	# copy into stage
-	rm -rf /tmp/test_load_unload_fs
-	echo "drop stage if exists data_fs;" | $BENDSQL_CLIENT_CONNECT
-	echo "create stage data_fs url = 'fs:///tmp/test_load_unload_fs/' FILE_FORMAT = (type = ${1});"  | $BENDSQL_CLIENT_CONNECT
-	echo "copy into @data_fs from test_load_unload file_format = (type = ${1});" | $BENDSQL_CLIENT_CONNECT
+	# load with copy into table
+	echo "copy into test_load_unload from @s1/unload1.parquet force=true;" | $BENDSQL_CLIENT_CONNECT
 
-	# unload clickhouse again from stage
-	curl -s -u root: -XPOST "http://localhost:${QUERY_CLICKHOUSE_HTTP_HANDLER_PORT}/?enable_clickhouse_handler=1" \
-	-d "select * from @data_fs FORMAT ${1}" > /tmp/test_load_unload4.parquet
+	# unload3
+	echo "copy into @s1/unload3/ from test_load_unload" | $BENDSQL_CLIENT_CONNECT
+	mv `ls /tmp/test_load_unload/unload3/*` /tmp/test_load_unload/unload3.parquet
 
-	diff /tmp/test_load_unload2.parquet /tmp/test_load_unload.parquet
-	diff /tmp/test_load_unload3.parquet /tmp/test_load_unload.parquet
-	diff /tmp/test_load_unload4.parquet /tmp/test_load_unload.parquet
+	diff /tmp/test_load_unload/unload2.parquet /tmp/test_load_unload/unload1.parquet
+	diff /tmp/test_load_unload/unload3.parquet /tmp/test_load_unload/unload1.parquet
 	echo "truncate table test_load_unload" | $BENDSQL_CLIENT_CONNECT
 }