Skip to content

Commit

Permalink
added native exclude filter handling via bitmaps
Browse files Browse the repository at this point in the history
  • Loading branch information
@glookka
glookka committed Jul 27, 2024
1 parent 3fb88e6 commit b707d5b
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 30 deletions.
45 changes: 24 additions & 21 deletions secondary/blockreader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ class SplitBitmap_c
FORCE_INLINE int Scan ( int iStart );
FORCE_INLINE int GetLength() const { return m_iSize; }

void Invert() { assert ( 0 && "Unsupported by SplitBitmap_c" ); }
void Invert ( int iMinBit=-1, int iMaxBit=-1 ) { assert ( 0 && "Unsupported by SplitBitmap_c" ); }

template <typename RESULT>
void Fetch ( int & iIterator, int iBase, RESULT * & pRes, RESULT * pMax );
Expand Down Expand Up @@ -203,7 +203,7 @@ class BitmapIterator_i : public BlockIterator_i
{
public:
virtual void Add ( BlockIterator_i * pIterator ) = 0;
virtual void Invert() = 0;
virtual void Invert ( RowidRange_t * pBounds ) = 0;
};

template <typename BITMAP, bool ROWID_RANGE>
Expand All @@ -221,7 +221,7 @@ class BitmapIterator_T : public BitmapIterator_i
bool WasCutoffHit() const override { return !m_iRowsLeft; }

void Add ( BlockIterator_i * pIterator ) override;
void Invert() override { m_tBitmap.Invert(); }
void Invert ( RowidRange_t * pBounds ) override { m_tBitmap.Invert ( pBounds ? pBounds->m_uMin : -1, pBounds ? pBounds->m_uMax : -1 ); }

private:
static const int RESULT_BLOCK_SIZE = 1024;
Expand Down Expand Up @@ -407,7 +407,7 @@ class ReaderTraits_c : public BlockReader_i
int m_iCutoff = 0;

bool NeedBitmapIterator() const;
BitmapIterator_i * SpawnBitmapIterator ( const RowidRange_t * pBounds = nullptr, const Filter_t * pRange = nullptr ) const;
BitmapIterator_i * SpawnBitmapIterator ( const RowidRange_t * pBounds, bool bExclude ) const;
void LoadValueBlockData ( bool bOnlyCount, FileReader_c & tReader );
uint32_t CalcNumBlockValues ( int iBlock ) const;
};
Expand Down Expand Up @@ -440,10 +440,11 @@ bool ReaderTraits_c::NeedBitmapIterator() const
}


BitmapIterator_i * ReaderTraits_c::SpawnBitmapIterator ( const RowidRange_t * pBounds, const Filter_t * pRange ) const
BitmapIterator_i * ReaderTraits_c::SpawnBitmapIterator ( const RowidRange_t * pBounds, bool bExclude ) const
{
// force bitmap iterator for IS NULL queries
if ( pRange && pRange->m_eType==common::FilterType_e::NOTNULL && pRange->m_bExclude )
// force bitmap iterator for exclude filters
// FIXME! make invertable split bitmaps
if ( bExclude )
{
if ( pBounds )
return new BitmapIterator_T<BitVec_T<uint64_t>, true> ( m_sAttr, m_tRsetInfo.m_uRowsCount, pBounds );
Expand Down Expand Up @@ -517,8 +518,8 @@ class BlockReader_c : public ReaderTraits_c
public:
BlockReader_c ( const ReaderFactory_c & tCtx, std::shared_ptr<IntCodec_i> & pCodec );

void CreateBlocksIterator ( const std::vector<BlockIter_t> & dIt, std::vector<BlockIterator_i *> & dRes ) override;
void CreateBlocksIterator ( const BlockIter_t & tIt, const Filter_t & tVal, std::vector<BlockIterator_i *> & dRes ) override { assert ( 0 && "Requesting range iterators from block reader" ); }
void CreateBlocksIterator ( const std::vector<BlockIter_t> & dIt, const Filter_t & tFilter, std::vector<BlockIterator_i *> & dRes ) override;
void CreateBlocksIterator ( const BlockIter_t & tIt, const Filter_t & tFilter, std::vector<BlockIterator_i *> & dRes ) override { assert ( 0 && "Requesting range iterators from block reader" ); }
uint32_t CalcValueCount ( const std::vector<BlockIter_t> & dIt ) override;
uint32_t CalcValueCount ( const BlockIter_t & tIt, const common::Filter_t & tVal ) override { assert ( 0 && "Requesting range iterators from block reader" ); return 0; }

Expand Down Expand Up @@ -639,17 +640,21 @@ void BlockReader_c::CreateBlocksIterator ( const BlockIter_t & tIt, ADDITERATOR
}


void BlockReader_c::CreateBlocksIterator ( const std::vector<BlockIter_t> & dIt, std::vector<BlockIterator_i *> & dRes )
void BlockReader_c::CreateBlocksIterator ( const std::vector<BlockIter_t> & dIt, const Filter_t & tFilter, std::vector<BlockIterator_i *> & dRes )
{
// add bitmap iterator as 1st element of dRes on exit
std::function<void( BitmapIterator_i * pIterator )> fnDeleter = [&]( BitmapIterator_i * pIterator ){ if ( pIterator ) { assert(dRes.empty()); dRes.push_back(pIterator); } };
std::unique_ptr<BitmapIterator_i, decltype(fnDeleter)> pBitmapIterator ( SpawnBitmapIterator(), fnDeleter );
RowidRange_t * pBounds = m_bHaveBounds ? &m_tBounds : nullptr;
std::unique_ptr<BitmapIterator_i, decltype(fnDeleter)> pBitmapIterator ( SpawnBitmapIterator ( pBounds, tFilter.m_bExclude ), fnDeleter );
if ( pBitmapIterator && m_iCutoff>=0 )
pBitmapIterator->SetCutoff(m_iCutoff);

std::unique_ptr<BlockIteratorWithSetup_i> pCommonIterator;
for ( auto & i : dIt )
CreateBlocksIterator ( i, [this, &dRes, &pBitmapIterator, &pCommonIterator]( int iItem ){ AddIterator ( iItem, dRes, pBitmapIterator.get(), pCommonIterator ); } );

if ( tFilter.m_bExclude )
pBitmapIterator->Invert(pBounds);
}


Expand Down Expand Up @@ -829,8 +834,8 @@ class RangeReader_c : public ReaderTraits_c
public:
RangeReader_c ( const ReaderFactory_c & tCtx, std::shared_ptr<IntCodec_i> & pCodec );

void CreateBlocksIterator ( const std::vector<BlockIter_t> & dIt, std::vector<BlockIterator_i *> & dRes ) override { assert ( 0 && "Requesting block iterators from range reader" ); }
void CreateBlocksIterator ( const BlockIter_t & tIt, const Filter_t & tRange, std::vector<BlockIterator_i *> & dRes ) override;
void CreateBlocksIterator ( const std::vector<BlockIter_t> & dIt, const Filter_t & tFilter, std::vector<BlockIterator_i *> & dRes ) override { assert ( 0 && "Requesting block iterators from range reader" ); }
void CreateBlocksIterator ( const BlockIter_t & tIt, const Filter_t & tFilter, std::vector<BlockIterator_i *> & dRes ) override;
uint32_t CalcValueCount ( const std::vector<BlockIter_t> & dIt ) override { assert ( 0 && "Requesting block iterators from range reader" ); return 0; }
uint32_t CalcValueCount ( const BlockIter_t & tIt, const common::Filter_t & tRange ) override;

Expand Down Expand Up @@ -1017,22 +1022,20 @@ void RangeReader_c::CreateBlocksIterator ( const BlockIter_t & tIt, const Filter
}


void RangeReader_c::CreateBlocksIterator ( const BlockIter_t & tIt, const Filter_t & tRange, std::vector<BlockIterator_i *> & dRes )
void RangeReader_c::CreateBlocksIterator ( const BlockIter_t & tIt, const Filter_t & tFilter, std::vector<BlockIterator_i *> & dRes )
{
// add bitmap iterator as 1st element of dRes on exit
std::function<void( BitmapIterator_i * pIterator )> fnDeleter = [&]( BitmapIterator_i * pIterator ){ if ( pIterator ) { assert(dRes.empty()); dRes.push_back(pIterator); } };
std::unique_ptr<BitmapIterator_i, decltype(fnDeleter)> pBitmapIterator ( SpawnBitmapIterator ( m_bHaveBounds ? &m_tBounds : nullptr, &tRange ), fnDeleter );
RowidRange_t * pBounds = m_bHaveBounds ? &m_tBounds : nullptr;
std::unique_ptr<BitmapIterator_i, decltype(fnDeleter)> pBitmapIterator ( SpawnBitmapIterator ( pBounds, tFilter.m_bExclude ), fnDeleter );
if ( pBitmapIterator && m_iCutoff>=0 )
pBitmapIterator->SetCutoff(m_iCutoff);

std::unique_ptr<BlockIteratorWithSetup_i> pCommonIterator;
CreateBlocksIterator ( tIt, tRange, [this, &dRes, &pBitmapIterator, &pCommonIterator]( int iValCur, bool bLoad ){ return AddIterator ( iValCur, bLoad, dRes, pBitmapIterator.get(), pCommonIterator ); } );
CreateBlocksIterator ( tIt, tFilter, [this, &dRes, &pBitmapIterator, &pCommonIterator]( int iValCur, bool bLoad ){ return AddIterator ( iValCur, bLoad, dRes, pBitmapIterator.get(), pCommonIterator ); } );

if ( tRange.m_eType==common::FilterType_e::NOTNULL && tRange.m_bExclude )
{
assert(pBitmapIterator);
pBitmapIterator->Invert();
}
if ( tFilter.m_bExclude )
pBitmapIterator->Invert(pBounds);
}


Expand Down
4 changes: 2 additions & 2 deletions secondary/blockreader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ class BlockReader_i
public:
virtual ~BlockReader_i() = default;

virtual void CreateBlocksIterator ( const std::vector<BlockIter_t> & dIt, std::vector<common::BlockIterator_i *> & dRes ) = 0;
virtual void CreateBlocksIterator ( const BlockIter_t & tIt, const common::Filter_t & tVal, std::vector<common::BlockIterator_i *> & dRes ) = 0;
virtual void CreateBlocksIterator ( const std::vector<BlockIter_t> & dIt, const common::Filter_t & tFilter, std::vector<common::BlockIterator_i *> & dRes ) = 0;
virtual void CreateBlocksIterator ( const BlockIter_t & tIt, const common::Filter_t & tFilter, std::vector<common::BlockIterator_i *> & dRes ) = 0;
virtual uint32_t CalcValueCount ( const std::vector<BlockIter_t> & dIt ) = 0;
virtual uint32_t CalcValueCount ( const BlockIter_t & tIt, const common::Filter_t & tVal ) = 0;
};
Expand Down
2 changes: 1 addition & 1 deletion secondary/secondary.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ int64_t SecondaryIndex_c::GetValsRows ( std::vector<BlockIterator_i *> * pIterat
if ( !pBlockReader )
return 0;

pBlockReader->CreateBlocksIterator ( dBlocksIt, *pIterators );
pBlockReader->CreateBlocksIterator ( dBlocksIt, tFilter, *pIterators );
return iNumIterators;
}

Expand Down
26 changes: 20 additions & 6 deletions util/bitvec.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,29 @@ class BitVec_T
m_dData [ iBit>>SHIFT ] |= ( (T)1 )<<( iBit&MASK );
}

void Invert()
void Invert ( int iMinBit=-1, int iMaxBit=-1 )
{
for ( auto & i : m_dData )
i = ~i;
if ( iMinBit<0 )
iMinBit = 0;

if ( m_iSize!=m_iDataLen << SHIFT )
if ( iMaxBit<0 )
iMaxBit = m_iSize;

int iMinId = iMinBit>>SHIFT;
int iMaxId = (iMaxBit+SIZEBITS-1)>>SHIFT;
for ( int i = iMinId; i<iMaxId; i++ )
m_dData[i] = ~m_dData[i];

if ( ( iMinId<<SHIFT ) != iMinBit )
{
int iFirstBit = iMinId << SHIFT;
m_dData[iMinId] &= ~((T(1) << ( iMinBit-iFirstBit )) - 1);
}

if ( ( iMaxId<<SHIFT ) != iMaxBit )
{
int iFirstBit = (m_iDataLen-1) << SHIFT;
m_dData.back() &= (T(1) << ( m_iSize-iFirstBit )) - 1;
int iFirstBit = (iMaxId-1) << SHIFT;
m_dData[iMaxId-1] &= (T(1) << ( iMaxBit-iFirstBit )) - 1;
}
}

Expand Down

0 comments on commit b707d5b

Please sign in to comment.