Merge branch 'master' into columnar-2.3.0

.github/workflows/build_template.yml

@@ -10,7 +10,7 @@ on:
       DISTR:
         required: false
         type: string
-        default: bionic
+        default: jammy
       CTEST_CMAKE_GENERATOR:
         required: false
         type: string

.github/workflows/pack_publish.yml

@@ -213,42 +213,21 @@ jobs:
       - id: vars
         run: |
           version=$( cat CMakeLists.txt | grep 'columnar VERSION' | cut -d' ' -f5  )
-          date=$( git log -1 --date=short --format=%cd | cut -c3- | tr -d '-' )
+          date=$( git log -1 --date=format-local:"%y%m%d%H" --format=%cd )
           commit=${GITHUB_SHA:0:7}
 
           echo "app_version=$version" >> $GITHUB_OUTPUT
           echo "app_date=$date" >> $GITHUB_OUTPUT
           echo "app_commit=$commit" >> $GITHUB_OUTPUT
 
   update-manticoresearch-deps:
-    name: Updating deps of manticoresearch
     needs:
       - vars
     runs-on: ubuntu-22.04
     steps:
-      - name: Checkout manticoresearch
-        uses: actions/checkout@v3
-        with:
-          repository: "manticoresoftware/manticoresearch"
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Update deps.txt
-        id: deps
-        run: |
-          version=${{ needs.vars.outputs.app_version }}
-          date=${{ needs.vars.outputs.app_date }}
-          commit=${{ needs.vars.outputs.app_commit }}
-
-          version_string="$version $date $commit"
-          echo "version_string=${version_string// /-}" >> $GITHUB_OUTPUT
-          sed -i '/^mcl\s/c\mcl '"$version_string" deps.txt
-
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v5
+      - name: Update deps
+        uses: manticoresoftware/manticoresearch/actions/update-deps@master
         with:
+          name: mcl
+          version: "${{ needs.vars.outputs.app_version }} ${{ needs.vars.outputs.app_date }} ${{ needs.vars.outputs.app_commit }}"
           token: ${{ secrets.PR_TOKEN }}
-          commit-message: "Bump columnar version to: ${{ steps.deps.outputs.version_string }}"
-          title: "Update columnar version"
-          body: "Updating columnar version to ${{ steps.deps.outputs.version_string }}"
-          branch: "update-columnar-version"
-          draft: false

.github/workflows/test.yml

@@ -1,5 +1,5 @@
 name: 🔬 Test
-run-name: 🔬 Test ${{ github.sha }}
+run-name: "🔬 Test ${{ github.event.schedule == '00 20 * * *' && 'master HEAD in daemon + ' || 'commit' }} ${{ github.sha }}"
 
 #on: workflow_call
 
@@ -58,7 +58,6 @@ jobs:
         run: |
           wget https://repo.manticoresearch.com/repository/ci/boost_1_75_0.tgz
           tar -xf boost_1_75_0.tgz
-
   build_linux_release:
     name: Linux release build
     uses: ./.github/workflows/build_template.yml

.github/workflows/test_template.yml

@@ -40,7 +40,7 @@ jobs:
       run:
         shell: bash
     container:
-      image: manticoresearch/ubertests_ctest:3263 #
+      image: manticoresearch/ubertests_ctest:3263_mar_2024
       env:
         DIAGNOSTIC: 1
         CACHEB: ../cache
@@ -62,7 +62,7 @@ jobs:
       - name: Download build artifacts
         uses: manticoresoftware/download_artifact_with_retries@v2
         with:
-          name: build_bionic_${{ inputs.CTEST_CONFIGURATION_TYPE }}_x86_64
+          name: build_jammy_${{ inputs.CTEST_CONFIGURATION_TYPE }}_x86_64
           path: .
 
       - name: Check out main cache before building

cmake/rev.cmake

@@ -11,6 +11,10 @@ function ( guess_from_git )
 		return ()
 	endif ()
 
+	# without this in some environments you can get error "detected dubious ownership in repository"
+	# `git config --global --add safe.directory '*'` in the docker image it runs in may not help. TODO: check why
+  execute_process ( COMMAND "${GIT_EXECUTABLE}" config --global --add safe.directory "${columnar_SOURCE_DIR}")
+
 	# extract short hash as GIT_COMMIT_ID
 	execute_process ( COMMAND "${GIT_EXECUTABLE}" log -1 --format=%h
 			WORKING_DIRECTORY "${columnar_SOURCE_DIR}"
@@ -21,16 +25,14 @@ function ( guess_from_git )
 	set ( GIT_COMMIT_ID "${GIT_COMMIT_ID}" PARENT_SCOPE )
 
 	# extract timestamp and make number YYMMDD from it
-	# it would be --date=format:%y%m%d, but old git on centos doesn't understand it
-	execute_process ( COMMAND "${GIT_EXECUTABLE}" log -1 --date=short --format=%cd
+	execute_process ( COMMAND "${GIT_EXECUTABLE}" log -1 --date=format-local:"%y%m%d%H" --format=%cd
 			WORKING_DIRECTORY "${columnar_SOURCE_DIR}"
 			RESULT_VARIABLE res
 			OUTPUT_VARIABLE GIT_TIMESTAMP_ID
-			ERROR_QUIET
 			OUTPUT_STRIP_TRAILING_WHITESPACE )
-	string ( REPLACE "-" "" GIT_TIMESTAMP_ID "${GIT_TIMESTAMP_ID}" )
-	string ( SUBSTRING "${GIT_TIMESTAMP_ID}" 2 -1 GIT_TIMESTAMP_ID )
-	set ( GIT_TIMESTAMP_ID "${GIT_TIMESTAMP_ID}" PARENT_SCOPE )
+
+	string ( SUBSTRING "${GIT_TIMESTAMP_ID}" 1 8 GIT_TIMESTAMP_ID )
+	set ( GIT_TIMESTAMP_ID ${GIT_TIMESTAMP_ID} PARENT_SCOPE )
 
 	# timestamp for reproducable packages
 	execute_process ( COMMAND "${GIT_EXECUTABLE}" log -1 --pretty=%ct

manticore_src.txt

@@ -1 +1 @@
-GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG 9052079ba
+GIT_REPOSITORY https://github.com/manticoresoftware/manticoresearch.git GIT_TAG 4a45ee194

secondary/builder.cpp

@@ -66,7 +66,7 @@ class RawWriter_i
 
 	virtual bool	Setup ( const std::string & sFile, const SchemaAttr_t & tAttr, int iAttr, std::string & sError ) = 0;
 	virtual int		GetItemSize () const = 0;
-	virtual void	SetItemsCount ( int iSize ) = 0;
+	virtual void	SetItemsCount ( size_t tSize ) = 0;
 
 	virtual void	SetAttr ( uint32_t tRowID, int64_t tAttr ) = 0;
 	virtual void	SetAttr ( uint32_t tRowID, const uint8_t * pData, int iLength ) = 0;
@@ -119,7 +119,7 @@ class RawWriter_T : public RawWriter_i
 
 	bool	Setup ( const std::string & sFile, const SchemaAttr_t & tAttr, int iAttr, std::string & sError ) final;
 	int		GetItemSize() const final { return sizeof ( m_dRows[0] ); }
-	void	SetItemsCount ( int iSize ) final { m_dRows.reserve ( iSize ); }
+	void	SetItemsCount ( size_t tSize ) final { m_dRows.reserve(tSize); }
 	void	Flush() final;
 	void	Done() final;
 	void	SetAttr ( uint32_t tRowID, int64_t tAttr ) final;
@@ -736,7 +736,7 @@ struct ScopedFilesRemoval_t
 class Builder_c final : public Builder_i
 {
 public:
-	bool	Setup ( const Settings_t & tSettings, const Schema_t & tSchema, int iMemoryLimit, const std::string & sFile, size_t tBufferSize, std::string & sError );
+	bool	Setup ( const Settings_t & tSettings, const Schema_t & tSchema, size_t tMemoryLimit, const std::string & sFile, size_t tBufferSize, std::string & sError );
 
 	void	SetRowID ( uint32_t tRowID ) final;
 	void	SetAttr ( int iAttr, int64_t tAttr ) final;
@@ -748,7 +748,7 @@ class Builder_c final : public Builder_i
 	std::string	m_sFile;
 	size_t		m_tBufferSize = 0;
 	uint32_t	m_tRowID = 0;
-	uint32_t	m_iMaxRows = 0;
+	uint32_t	m_uMaxRows = 0;
 
 	std::vector<std::shared_ptr<RawWriter_i>>	m_dRawWriter;
 	std::vector<std::shared_ptr<SIWriter_i>>	m_dCidWriter;
@@ -761,7 +761,7 @@ class Builder_c final : public Builder_i
 };
 
 
-bool Builder_c::Setup ( const Settings_t & tSettings, const Schema_t & tSchema, int iMemoryLimit, const std::string & sFile, size_t tBufferSize, std::string & sError )
+bool Builder_c::Setup ( const Settings_t & tSettings, const Schema_t & tSchema, size_t tMemoryLimit, const std::string & sFile, size_t tBufferSize, std::string & sError )
 {
 	m_sFile = sFile;
 	m_tBufferSize = tBufferSize;
@@ -823,14 +823,14 @@ bool Builder_c::Setup ( const Settings_t & tSettings, const Schema_t & tSchema,
 		if ( pWriter )
 			iRowSize += pWriter->GetItemSize();
 
-	m_iMaxRows = std::max ( 1000, iMemoryLimit / iRowSize );
+	m_uMaxRows = std::min ( std::max ( size_t(10000), tMemoryLimit / iRowSize ), size_t(UINT32_MAX) );
 
 	for ( auto & pWriter : m_dRawWriter )
 	{
 		if ( !pWriter )
 			continue;
 
-		pWriter->SetItemsCount ( m_iMaxRows );
+		pWriter->SetItemsCount ( m_uMaxRows );
 	}
 
 	return true;
@@ -840,7 +840,7 @@ void Builder_c::SetRowID ( uint32_t tRowID )
 {
 	m_tRowID = tRowID;
 
-	if ( ( m_tRowID % m_iMaxRows )==0 )
+	if ( ( m_tRowID % m_uMaxRows )==0 )
 		Flush();
 }
 
@@ -1035,11 +1035,11 @@ RawValue_T<uint64_t> Convert ( const BinValue_T<uint64_t> & tSrc )
 } // namespace SI
 
 
-SI::Builder_i * CreateBuilder ( const Schema_t & tSchema, int iMemoryLimit, const std::string & sFile, size_t tBufferSize, std::string & sError )
+SI::Builder_i * CreateBuilder ( const Schema_t & tSchema, size_t tMemoryLimit, const std::string & sFile, size_t tBufferSize, std::string & sError )
 {
 	std::unique_ptr<SI::Builder_c> pBuilder ( new SI::Builder_c );
 	SI::Settings_t tSettings;
-	if ( !pBuilder->Setup ( tSettings, tSchema, iMemoryLimit, sFile, tBufferSize, sError ) )
+	if ( !pBuilder->Setup ( tSettings, tSchema, tMemoryLimit, sFile, tBufferSize, sError ) )
 		return nullptr;
 
 	return pBuilder.release();

secondary/builder.h

@@ -43,5 +43,5 @@ class Builder_i
 
 extern "C"
 {
-	DLLEXPORT SI::Builder_i * CreateBuilder ( const common::Schema_t & tSchema, int iMemoryLimit, const std::string & sFile, size_t tBufferSize, std::string & sError );
+	DLLEXPORT SI::Builder_i * CreateBuilder ( const common::Schema_t & tSchema, size_t tMemoryLimit, const std::string & sFile, size_t tBufferSize, std::string & sError );
 }

secondary/secondary.cpp

@@ -82,7 +82,7 @@ bool SecondaryIndex_c::Setup ( const std::string & sFile, std::string & sError )
 	m_uVersion = m_tReader.Read_uint32();
 
 	// starting with v.6 we have backward compatibility
-	if ( m_uVersion<6 )
+	if ( m_uVersion<6 || m_uVersion>STORAGE_VERSION )
 	{
 		sError = FormatStr ( "Unable to load inverted index: %s is v.%d, binary is v.%d", sFile.c_str(), m_uVersion, STORAGE_VERSION );
 		return false;

secondary/secondary.h

@@ -38,7 +38,7 @@ namespace common
 namespace SI
 {
 
-static const int LIB_VERSION = 14;
+static const int LIB_VERSION = 15;
 static const uint32_t STORAGE_VERSION = 8;
 
 class Index_i