00001 #ifndef BMFUNC__H__INCLUDED__
00002 #define BMFUNC__H__INCLUDED__
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029 #include <memory.h>
00030
00031 #include "bmdef.h"
00032 #include "bmutil.h"
00033
00034
00035 #ifdef _MSC_VER
00036 # pragma warning( disable: 4146 )
00037 #endif
00038
00039 namespace bm
00040 {
00041
00042
00043
00044
00045
00046
00047
00048 struct bv_statistics
00049 {
00050
00051 unsigned bit_blocks;
00052
00053 unsigned gap_blocks;
00054
00055 unsigned max_serialize_mem;
00056
00057 unsigned memory_used;
00058
00059 gap_word_t gap_length[bm::set_total_blocks];
00060
00061 gap_word_t gap_levels[bm::gap_levels];
00062
00063
00064
00065
00066 void add_bit_block()
00067 {
00068 ++bit_blocks;
00069 unsigned mem_used = sizeof(bm::word_t) * bm::set_block_size;
00070 memory_used += mem_used;
00071 max_serialize_mem += mem_used;
00072 }
00073
00074
00075 void add_gap_block(unsigned capacity, unsigned length)
00076 {
00077 ++gap_blocks;
00078 unsigned mem_used = capacity * sizeof(gap_word_t);
00079 memory_used += mem_used;
00080 max_serialize_mem += length * sizeof(gap_word_t);
00081 }
00082 };
00083
00084
00085
00086
00087
00088
00089
00090
00091
00092
00093
00094
00095
00096
00097
00098
00099
00100
00101
00102 template<bool T> struct gap_len_table
00103 {
00104 static const gap_word_t _len[bm::gap_levels];
00105 };
00106
00107 template<bool T>
00108 const gap_word_t gap_len_table<T>::_len[bm::gap_levels] =
00109 { 128, 256, 512, bm::gap_max_buff_len };
00110
00111
00112
00113
00114
00115
00116
00117 template<bool T> struct gap_len_table_min
00118 {
00119 static const gap_word_t _len[bm::gap_levels];
00120 };
00121
00122 template<bool T>
00123 const gap_word_t gap_len_table_min<T>::_len[bm::gap_levels] =
00124 { 32, 96, 128, 512 };
00125
00126
00127
00128
00129
00130
00131
00132
00133 template<bool T> struct block_set_table
00134 {
00135 static const unsigned _left[32];
00136 static const unsigned _right[32];
00137 };
00138
00139 template<bool T>
00140 const unsigned block_set_table<T>::_left[32] = {
00141 0x1, 0x3, 0x7, 0xf, 0x1f, 0x3f, 0x7f, 0xff, 0x1ff, 0x3ff, 0x7ff,
00142 0xfff, 0x1fff, 0x3fff, 0x7fff, 0xffff, 0x1ffff, 0x3ffff, 0x7ffff,
00143 0xfffff, 0x1fffff, 0x3fffff, 0x7fffff, 0xffffff, 0x1ffffff, 0x3ffffff,
00144 0x7ffffff, 0xfffffff, 0x1fffffff, 0x3fffffff, 0x7fffffff, 0xffffffff
00145 };
00146
00147 template<bool T>
00148 const unsigned block_set_table<T>::_right[32] = {
00149 0xffffffff, 0xfffffffe, 0xfffffffc, 0xfffffff8, 0xfffffff0,
00150 0xffffffe0, 0xffffffc0, 0xffffff80, 0xffffff00, 0xfffffe00,
00151 0xfffffc00, 0xfffff800, 0xfffff000, 0xffffe000, 0xffffc000,
00152 0xffff8000, 0xffff0000, 0xfffe0000, 0xfffc0000, 0xfff80000,
00153 0xfff00000, 0xffe00000, 0xffc00000, 0xff800000, 0xff000000,
00154 0xfe000000, 0xfc000000, 0xf8000000, 0xf0000000, 0xe0000000,
00155 0xc0000000, 0x80000000
00156 };
00157
00158
00159
00160
00161
00162
00163
00164 BMFORCEINLINE
00165 bm::id_t word_bitcount(bm::id_t w)
00166 {
00167 #ifdef BMSSE4OPT
00168 return _mm_popcnt_u32(w);
00169 #else
00170 return
00171 bm::bit_count_table<true>::_count[(unsigned char)(w)] +
00172 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 8)] +
00173 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 16)] +
00174 bm::bit_count_table<true>::_count[(unsigned char)((w) >> 24)];
00175 #endif
00176 }
00177
00178 inline
00179 int parallel_popcnt_32(unsigned int n)
00180 {
00181 unsigned int tmp;
00182
00183 tmp = n - ((n >> 1) & 033333333333)
00184 - ((n >> 2) & 011111111111);
00185 return ((tmp + (tmp >> 3)) & 030707070707) % 63;
00186 }
00187
00188 #ifdef BM64OPT
00189
00190
00191
00192
00193
00194 inline
00195 int word_bitcount64(bm::id64_t x)
00196 {
00197 x = x - ((x >> 1) & 0x5555555555555555);
00198 x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333);
00199 x = (x + (x >> 4)) & 0x0F0F0F0F0F0F0F0F;
00200 x = x + (x >> 8);
00201 x = x + (x >> 16);
00202 x = x + (x >> 32);
00203 return x & 0xFF;
00204 }
00205
00206 inline
00207 unsigned bitcount64_4way(bm::id64_t x, bm::id64_t y,
00208 bm::id64_t u, bm::id64_t v)
00209 {
00210 const bm::id64_t m1 = 0x5555555555555555;
00211 const bm::id64_t m2 = 0x3333333333333333;
00212 const bm::id64_t m3 = 0x0F0F0F0F0F0F0F0F;
00213 const bm::id64_t m4 = 0x000000FF000000FF;
00214
00215 x = x - ((x >> 1) & m1);
00216 y = y - ((y >> 1) & m1);
00217 u = u - ((u >> 1) & m1);
00218 v = v - ((v >> 1) & m1);
00219 x = (x & m2) + ((x >> 2) & m2);
00220 y = (y & m2) + ((y >> 2) & m2);
00221 u = (u & m2) + ((u >> 2) & m2);
00222 v = (v & m2) + ((v >> 2) & m2);
00223 x = x + y;
00224 u = u + v;
00225 x = (x & m3) + ((x >> 4) & m3);
00226 u = (u & m3) + ((u >> 4) & m3);
00227 x = x + u;
00228 x = x + (x >> 8);
00229 x = x + (x >> 16);
00230 x = x & m4;
00231 x = x + (x >> 32);
00232 return x & 0x000001FF;
00233 }
00234
00235
00236 #endif
00237
00238
00239
00240
00241
00242
00243
00244
00245 enum set_operation
00246 {
00247 set_AND = 0,
00248 set_OR = 1,
00249 set_SUB = 2,
00250 set_XOR = 3,
00251 set_ASSIGN = 4,
00252 set_COUNT = 5,
00253 set_COUNT_AND = 6,
00254 set_COUNT_XOR = 7,
00255 set_COUNT_OR = 8,
00256 set_COUNT_SUB_AB= 9,
00257 set_COUNT_SUB_BA= 10,
00258 set_COUNT_A = 11,
00259 set_COUNT_B = 12,
00260
00261 set_END
00262 };
00263
00264
00265 inline
00266 bool is_const_set_operation(set_operation op)
00267 {
00268 return (int(op) >= int(set_COUNT));
00269 }
00270
00271
00272
00273
00274 enum operation
00275 {
00276 BM_AND = set_AND,
00277 BM_OR = set_OR,
00278 BM_SUB = set_SUB,
00279 BM_XOR = set_XOR
00280 };
00281
00282
00283
00284
00285 inline
00286 bm::operation setop2op(bm::set_operation op)
00287 {
00288 BM_ASSERT(op == set_AND ||
00289 op == set_OR ||
00290 op == set_SUB ||
00291 op == set_XOR);
00292 return (bm::operation) op;
00293 }
00294
00295
00296
00297
00298
00299
00300
00301 template<bool T> struct all_set
00302 {
00303 struct BM_ALIGN16 all_set_block
00304 {
00305 bm::word_t _p[bm::set_block_size] BM_ALIGN16ATTR;
00306
00307 all_set_block()
00308 {
00309 ::memset(_p, 0xFF, sizeof(_p));
00310 }
00311 };
00312
00313 static all_set_block _block;
00314 };
00315
00316
00317 template<bool T> typename all_set<T>::all_set_block all_set<T>::_block;
00318
00319
00320 template<typename W>
00321 void xor_swap(W& x, W& y)
00322 {
00323 BM_ASSERT(&x != &y);
00324 x ^= y;
00325 y ^= x;
00326 x ^= y;
00327 }
00328
00329
00330
00331
00332
00333
00334
00335
00336
00337
00338
00339
00340
00341 template<typename T> int wordcmp0(T w1, T w2)
00342 {
00343 while (w1 != w2)
00344 {
00345 int res = (w1 & 1) - (w2 & 1);
00346 if (res != 0) return res;
00347 w1 >>= 1;
00348 w2 >>= 1;
00349 }
00350 return 0;
00351 }
00352
00353
00354
00355
00356
00357
00358
00359
00360
00361
00362
00363
00364
00365
00366
00367
00368
00369
00370
00371 template<typename T> int wordcmp(T a, T b)
00372 {
00373 T diff = a ^ b;
00374 return diff? ( (a & diff & -diff)? 1 : -1 ) : 0;
00375 }
00376
00377
00378
00379
00380
00381
00382
00383
00384
00385 template<bool T> struct _copyright
00386 {
00387 static const char _p[];
00388 };
00389
00390 template<bool T> const char _copyright<T>::_p[] =
00391 "BitMagic C++ Library. v.3.6.4 (c) 2002-2010 Anatoliy Kuznetsov.";
00392
00393
00394
00395
00396
00397 enum ByteOrder
00398 {
00399 BigEndian = 0,
00400 LittleEndian = 1
00401 };
00402
00403
00404
00405
00406
00407 template<bool T> struct globals
00408 {
00409 struct bo
00410 {
00411 ByteOrder _byte_order;
00412
00413 bo()
00414 {
00415 unsigned x;
00416 unsigned char *s = (unsigned char *)&x;
00417 s[0] = 1;
00418 s[1] = 2;
00419 s[2] = 3;
00420 s[3] = 4;
00421
00422 if(x == 0x04030201)
00423 {
00424 _byte_order = LittleEndian;
00425 return;
00426 }
00427
00428 if(x == 0x01020304)
00429 {
00430 _byte_order = BigEndian;
00431 return;
00432 }
00433
00434 BM_ASSERT(0);
00435 _byte_order = LittleEndian;
00436 }
00437 };
00438
00439 static bo _bo;
00440
00441 static ByteOrder byte_order() { return _bo._byte_order; }
00442
00443 };
00444
00445 template<bool T> typename globals<T>::bo globals<T>::_bo;
00446
00447
00448
00449
00450
00451
00452
00453
00454
00455
00456
00457
00458
00459
00460 template<typename T>
00461 unsigned gap_bfind(const T* buf, unsigned pos, unsigned* is_set)
00462 {
00463 BM_ASSERT(pos < bm::gap_max_bits);
00464 *is_set = (*buf) & 1;
00465
00466 register unsigned start = 1;
00467 register unsigned end = 1 + ((*buf) >> 3);
00468
00469 while ( start != end )
00470 {
00471 unsigned curr = (start + end) >> 1;
00472 if ( buf[curr] < pos )
00473 start = curr + 1;
00474 else
00475 end = curr;
00476 }
00477 *is_set ^= ((start-1) & 1);
00478 return start;
00479 }
00480
00481
00482
00483
00484
00485
00486
00487
00488
00489 template<typename T> unsigned gap_test(const T* buf, unsigned pos)
00490 {
00491 BM_ASSERT(pos < bm::gap_max_bits);
00492
00493 unsigned start = 1;
00494 unsigned end = 1 + ((*buf) >> 3);
00495
00496 if (end - start < 10)
00497 {
00498 unsigned sv = *buf & 1;
00499 unsigned sv1= sv ^ 1;
00500 if (buf[1] >= pos) return sv;
00501 if (buf[2] >= pos) return sv1;
00502 if (buf[3] >= pos) return sv;
00503 if (buf[4] >= pos) return sv1;
00504 if (buf[5] >= pos) return sv;
00505 if (buf[6] >= pos) return sv1;
00506 if (buf[7] >= pos) return sv;
00507 if (buf[8] >= pos) return sv1;
00508 if (buf[9] >= pos) return sv;
00509 BM_ASSERT(0);
00510 }
00511 else
00512 while ( start != end )
00513 {
00514 unsigned curr = (start + end) >> 1;
00515 if ( buf[curr] < pos )
00516 start = curr + 1;
00517 else
00518 end = curr;
00519 }
00520 return ((*buf) & 1) ^ ((--start) & 1);
00521 }
00522
00523
00524
00525
00526 template<class T, class F>
00527 void for_each_nzblock(T*** root, unsigned size1, unsigned size2,
00528 F& f)
00529 {
00530 unsigned block_idx = 0;
00531 for (unsigned i = 0; i < size1; ++i)
00532 {
00533 T** blk_blk = root[i];
00534
00535 if (!blk_blk)
00536 {
00537 f.on_empty_top(i);
00538 block_idx += size2;
00539 continue;
00540 }
00541
00542 unsigned non_empty_top = 0;
00543 for (unsigned j = 0;j < size2; ++j, ++block_idx)
00544 {
00545 if (blk_blk[j])
00546 {
00547 f(blk_blk[j], block_idx);
00548
00549 non_empty_top += (blk_blk[j] != 0);
00550 }
00551 else
00552 {
00553 f.on_empty_block(block_idx);
00554 }
00555 }
00556 if (non_empty_top == 0)
00557 {
00558 f.on_empty_top(i);
00559 }
00560 }
00561 }
00562
00563
00564
00565
00566 template<class T, class F>
00567 bool for_each_nzblock_if(T*** root, unsigned size1, unsigned size2, F& f)
00568 {
00569 unsigned block_idx = 0;
00570 for (unsigned i = 0; i < size1; ++i)
00571 {
00572 T** blk_blk = root[i];
00573
00574 if (!blk_blk)
00575 {
00576 block_idx += bm::set_array_size;
00577 continue;
00578 }
00579
00580 for (unsigned j = 0;j < size2; ++j, ++block_idx)
00581 {
00582 if (blk_blk[j])
00583 if (f(blk_blk[j], block_idx)) return true;
00584 }
00585 }
00586 return false;
00587 }
00588
00589
00590
00591 template<class T, class F>
00592 void for_each_block(T*** root, unsigned size1, unsigned size2, F& f)
00593 {
00594 unsigned block_idx = 0;
00595
00596 for (unsigned i = 0; i < size1; ++i)
00597 {
00598 T** blk_blk = root[i];
00599
00600 if (blk_blk)
00601 {
00602 for (unsigned j = 0;j < size2; ++j, ++block_idx)
00603 {
00604 f(blk_blk[j], block_idx);
00605 }
00606 }
00607 else
00608 {
00609 for (unsigned j = 0;j < size2; ++j, ++block_idx)
00610 {
00611 f(0, block_idx);
00612 }
00613 }
00614 }
00615 }
00616
00617
00618
00619
00620
00621 template<class T, class F> F bmfor_each(T first, T last, F f)
00622 {
00623 do
00624 {
00625 f(*first);
00626 ++first;
00627 } while (first < last);
00628 return f;
00629 }
00630
00631
00632
00633 template<class T> T sum_arr(T* first, T* last)
00634 {
00635 T sum = 0;
00636 while (first < last)
00637 {
00638 sum += *first;
00639 ++first;
00640 }
00641 return sum;
00642 }
00643
00644
00645
00646
00647
00648
00649
00650
00651
00652
00653 template<typename T> unsigned gap_bit_count(const T* buf, unsigned dsize=0)
00654 {
00655 register const T* pcurr = buf;
00656 if (dsize == 0)
00657 dsize = (*pcurr >> 3);
00658
00659 register const T* pend = pcurr + dsize;
00660
00661 register unsigned bits_counter = 0;
00662 ++pcurr;
00663
00664 if (*buf & 1)
00665 {
00666 bits_counter += *pcurr + 1;
00667 ++pcurr;
00668 }
00669 ++pcurr;
00670
00671 while (pcurr <= pend)
00672 {
00673 bits_counter += *pcurr - *(pcurr-1);
00674 pcurr += 2;
00675 }
00676
00677 return bits_counter;
00678 }
00679
00680
00681
00682
00683
00684
00685
00686
00687
00688 template<typename T>
00689 unsigned gap_bit_count_range(const T* buf, T left, T right)
00690 {
00691 BM_ASSERT(left <= right);
00692
00693 const T* pcurr = buf;
00694 const T* pend = pcurr + (*pcurr >> 3);
00695
00696 unsigned bits_counter = 0;
00697 unsigned is_set;
00698 unsigned start_pos = gap_bfind(buf, left, &is_set);
00699
00700 pcurr = buf + start_pos;
00701 if (right <= *pcurr)
00702 {
00703 if (is_set)
00704 bits_counter = (right - left + 1);
00705 return bits_counter;
00706 }
00707 if (is_set)
00708 bits_counter += *pcurr - left + 1;
00709
00710 unsigned prev_gap = *pcurr++;
00711 is_set ^= 1;
00712 while (right > *pcurr)
00713 {
00714 if (is_set)
00715 bits_counter += *pcurr - prev_gap;
00716 if (pcurr == pend)
00717 return bits_counter;
00718 prev_gap = *pcurr++;
00719 is_set ^= 1;
00720 }
00721 if (is_set)
00722 bits_counter += right - prev_gap;
00723
00724 return bits_counter;
00725 }
00726
00727
00728
00729
00730
00731
00732
00733
00734
00735 template<class T, class Func>
00736 void for_each_dgap(const T* gap_buf, Func& func)
00737 {
00738 const T* pcurr = gap_buf;
00739 const T* pend = pcurr + (*pcurr >> 3);
00740 ++pcurr;
00741
00742 T prev = *pcurr;
00743 func(prev + 1);
00744 ++pcurr;
00745 do
00746 {
00747 func(*pcurr - prev);
00748 prev = *pcurr;
00749 } while (++pcurr < pend);
00750 }
00751
00752
00753
00754
00755 template<typename T> struct d_copy_func
00756 {
00757 d_copy_func(T* dg_buf) : dgap_buf_(dg_buf) {}
00758 void operator()(T dgap) { *dgap_buf_++ = dgap; }
00759
00760 T* dgap_buf_;
00761 };
00762
00763
00764
00765
00766
00767
00768
00769
00770
00771
00772
00773
00774
00775
00776 template<typename T>
00777 T* gap_2_dgap(const T* gap_buf, T* dgap_buf, bool copy_head=true)
00778 {
00779 if (copy_head)
00780 {
00781 *dgap_buf++ = *gap_buf;
00782 }
00783
00784 d_copy_func<T> copy_func(dgap_buf);
00785 for_each_dgap<T, d_copy_func<T> >(gap_buf, copy_func);
00786 return copy_func.dgap_buf_;
00787 }
00788
00789
00790
00791
00792
00793
00794
00795
00796
00797
00798
00799
00800 template<typename T>
00801 void dgap_2_gap(const T* dgap_buf, T* gap_buf, T gap_header=0)
00802 {
00803 register const T* pcurr = dgap_buf;
00804 unsigned len;
00805 if (!gap_header)
00806 {
00807 len = *pcurr >> 3;
00808 *gap_buf++ = *pcurr++;
00809 }
00810 else
00811 {
00812 len = gap_header >> 3;
00813 *gap_buf++ = gap_header;
00814 }
00815 --len;
00816 register const T* pend = pcurr + len;
00817
00818 *gap_buf = *pcurr++;
00819 if (*gap_buf == 0)
00820 *gap_buf = 65535;
00821 else
00822 *gap_buf = *gap_buf - 1;
00823
00824 for (++gap_buf; pcurr < pend; ++pcurr)
00825 {
00826 T prev = *(gap_buf-1);
00827 *gap_buf++ = *pcurr + prev;
00828 }
00829 *gap_buf = 65535;
00830 }
00831
00832
00833
00834
00835
00836
00837
00838
00839
00840
00841 template<typename T> int gapcmp(const T* buf1, const T* buf2)
00842 {
00843 const T* pcurr1 = buf1;
00844 const T* pend1 = pcurr1 + (*pcurr1 >> 3);
00845 unsigned bitval1 = *buf1 & 1;
00846 ++pcurr1;
00847
00848 const T* pcurr2 = buf2;
00849 unsigned bitval2 = *buf2 & 1;
00850 ++pcurr2;
00851
00852 while (pcurr1 <= pend1)
00853 {
00854 if (*pcurr1 == *pcurr2)
00855 {
00856 if (bitval1 != bitval2)
00857 {
00858 return (bitval1) ? 1 : -1;
00859 }
00860 }
00861 else
00862 {
00863 if (bitval1 == bitval2)
00864 {
00865 if (bitval1)
00866 {
00867 return (*pcurr1 < *pcurr2) ? -1 : 1;
00868 }
00869 else
00870 {
00871 return (*pcurr1 < *pcurr2) ? 1 : -1;
00872 }
00873 }
00874 else
00875 {
00876 return (bitval1) ? 1 : -1;
00877 }
00878 }
00879
00880 ++pcurr1; ++pcurr2;
00881
00882 bitval1 ^= 1;
00883 bitval2 ^= 1;
00884 }
00885
00886 return 0;
00887 }
00888
00889
00890
00891
00892
00893
00894
00895
00896
00897
00898
00899
00900
00901
00902
00903
00904
00905
00906
00907 template<typename T, class F>
00908 void gap_buff_op(T* BMRESTRICT dest,
00909 const T* BMRESTRICT vect1,
00910 unsigned vect1_mask,
00911 const T* BMRESTRICT vect2,
00912 unsigned vect2_mask,
00913 F& f,
00914 unsigned& dlen)
00915 {
00916 register const T* cur1 = vect1;
00917 register const T* cur2 = vect2;
00918
00919 unsigned bitval1 = (*cur1++ & 1) ^ vect1_mask;
00920 unsigned bitval2 = (*cur2++ & 1) ^ vect2_mask;
00921
00922 unsigned bitval = f(bitval1, bitval2);
00923 unsigned bitval_prev = bitval;
00924
00925 register T* res = dest;
00926 *res = bitval;
00927 ++res;
00928
00929 while (1)
00930 {
00931 bitval = f(bitval1, bitval2);
00932
00933
00934
00935 if (bitval != bitval_prev)
00936 {
00937 ++res;
00938 bitval_prev = bitval;
00939 }
00940
00941 if (*cur1 < *cur2)
00942 {
00943 *res = *cur1;
00944 ++cur1;
00945 bitval1 ^= 1;
00946 }
00947 else
00948 {
00949 *res = *cur2;
00950 if (*cur2 < *cur1)
00951 {
00952 bitval2 ^= 1;
00953 }
00954 else
00955 {
00956 if (*cur2 == (bm::gap_max_bits - 1))
00957 {
00958 break;
00959 }
00960
00961 ++cur1;
00962 bitval1 ^= 1;
00963 bitval2 ^= 1;
00964 }
00965 ++cur2;
00966 }
00967
00968 }
00969
00970 dlen = (unsigned)(res - dest);
00971 *dest = (*dest & 7) + (dlen << 3);
00972
00973 }
00974
00975
00976
00977
00978
00979
00980
00981
00982
00983
00984
00985
00986
00987
00988
00989 template<typename T, class F>
00990 unsigned gap_buff_any_op(const T* BMRESTRICT vect1,
00991 unsigned vect1_mask,
00992 const T* BMRESTRICT vect2,
00993 unsigned vect2_mask,
00994 F f)
00995 {
00996 register const T* cur1 = vect1;
00997 register const T* cur2 = vect2;
00998
00999 unsigned bitval1 = (*cur1++ & 1) ^ vect1_mask;
01000 unsigned bitval2 = (*cur2++ & 1) ^ vect2_mask;
01001
01002 unsigned bitval = f(bitval1, bitval2);
01003 if (bitval)
01004 return bitval;
01005 unsigned bitval_prev = bitval;
01006
01007 while (1)
01008 {
01009 bitval = f(bitval1, bitval2);
01010 if (bitval)
01011 return bitval;
01012
01013 if (bitval != bitval_prev)
01014 bitval_prev = bitval;
01015
01016 if (*cur1 < *cur2)
01017 {
01018 ++cur1;
01019 bitval1 ^= 1;
01020 }
01021 else
01022 {
01023 if (*cur2 < *cur1)
01024 {
01025 bitval2 ^= 1;
01026 }
01027 else
01028 {
01029 if (*cur2 == (bm::gap_max_bits - 1))
01030 {
01031 break;
01032 }
01033
01034 ++cur1;
01035 bitval1 ^= 1;
01036 bitval2 ^= 1;
01037 }
01038 ++cur2;
01039 }
01040
01041 }
01042
01043 return 0;
01044 }
01045
01046
01047
01048
01049
01050
01051
01052
01053
01054
01055
01056
01057
01058
01059
01060
01061
01062
01063
01064
01065
01066
01067
01068
01069
01070
01071
01072
01073
01074
01075
01076
01077
01078
01079
01080
01081
01082
01083
01084
01085
01086
01087
01088
01089
01090
01091
01092
01093
01094
01095
01096
01097
01098
01099
01100
01101
01102
01103
01104
01105
01106
01107
01108
01109
01110
01111
01112
01113
01114
01115
01116
01117
01118
01119
01120
01121
01122
01123
01124
01125
01126
01127
01128
01129
01130
01131
01132
01133 template<typename T> unsigned gap_set_value(unsigned val,
01134 T* BMRESTRICT buf,
01135 unsigned pos,
01136 unsigned* BMRESTRICT is_set)
01137 {
01138 BM_ASSERT(pos < bm::gap_max_bits);
01139 unsigned curr = gap_bfind(buf, pos, is_set);
01140
01141 register T end = (*buf >> 3);
01142 if (*is_set == val)
01143 {
01144 *is_set = 0;
01145 return end;
01146 }
01147 *is_set = 1;
01148
01149 register T* pcurr = buf + curr;
01150 register T* pprev = pcurr - 1;
01151 register T* pend = buf + end;
01152
01153
01154
01155 if (pos == 0)
01156 {
01157 *buf ^= 1;
01158 if ( buf[1] )
01159 {
01160 ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
01161 buf[1] = 0;
01162 ++end;
01163 }
01164 else
01165 {
01166 pprev = buf + 1;
01167 pcurr = pprev + 1;
01168 do
01169 {
01170 *pprev++ = *pcurr++;
01171 } while (pcurr < pend);
01172 --end;
01173 }
01174 }
01175 else if (curr > 1 && ((unsigned)(*pprev))+1 == pos)
01176 {
01177 ++(*pprev);
01178 if (*pprev == *pcurr)
01179 {
01180 --end;
01181 if (pcurr != pend)
01182 {
01183 --end;
01184 ++pcurr;
01185 do
01186 {
01187 *pprev++ = *pcurr++;
01188 } while (pcurr < pend);
01189 }
01190 }
01191 }
01192 else if (*pcurr == pos)
01193 {
01194 --(*pcurr);
01195 if (pcurr == pend)
01196 {
01197 ++end;
01198 }
01199 }
01200 else
01201 {
01202 ::memmove(pcurr+2, pcurr,(end - curr + 1)*sizeof(T));
01203 *pcurr++ = pos - 1;
01204 *pcurr = pos;
01205 end+=2;
01206 }
01207
01208
01209 *buf = (*buf & 7) + (end << 3);
01210
01211 buf[end] = bm::gap_max_bits - 1;
01212 return end;
01213 }
01214
01215
01216
01217
01218
01219
01220
01221
01222
01223
01224
01225 template<typename T>
01226 unsigned gap_add_value(T* buf, unsigned pos)
01227 {
01228 BM_ASSERT(pos < bm::gap_max_bits);
01229
01230 register T end = (*buf >> 3);
01231 T curr = end;
01232 register T* pcurr = buf + end;
01233 register T* pend = pcurr;
01234 register T* pprev = pcurr - 1;
01235
01236
01237
01238 if (pos == 0)
01239 {
01240 *buf ^= 1;
01241 if ( buf[1] )
01242 {
01243 ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
01244 buf[1] = 0;
01245 ++end;
01246 }
01247 else
01248 {
01249 pprev = buf + 1;
01250 pcurr = pprev + 1;
01251 do
01252 {
01253 *pprev++ = *pcurr++;
01254 } while (pcurr < pend);
01255 --end;
01256 }
01257 }
01258 else if (((unsigned)(*pprev))+1 == pos && (curr > 1) )
01259 {
01260 ++(*pprev);
01261 if (*pprev == *pcurr)
01262 {
01263 --end;
01264 if (pcurr != pend)
01265 {
01266
01267 --end;
01268 ++pcurr;
01269 do
01270 {
01271 *pprev++ = *pcurr++;
01272 } while (pcurr < pend);
01273 }
01274 }
01275 }
01276 else if (*pcurr == pos)
01277 {
01278 --(*pcurr);
01279 if (pcurr == pend)
01280 {
01281 ++end;
01282 }
01283 }
01284 else
01285 {
01286 *pcurr++ = pos - 1;
01287 *pcurr = pos;
01288 end+=2;
01289 }
01290
01291
01292 *buf = (*buf & 7) + (end << 3);
01293
01294 buf[end] = bm::gap_max_bits - 1;
01295 return end;
01296 }
01297
01298
01299
01300
01301
01302
01303
01304
01305
01306
01307
01308
01309
01310 template<typename T>
01311 unsigned gap_set_array(T* buf, const T* arr, unsigned len)
01312 {
01313 *buf = (*buf & 6u) + (1u << 3);
01314
01315 T* pcurr = buf + 1;
01316
01317 unsigned i = 0;
01318 T curr = arr[i];
01319 if (curr != 0)
01320 {
01321 *pcurr = curr - 1;
01322 ++pcurr;
01323 }
01324 else
01325 {
01326 *buf += 1;
01327 }
01328 T prev = curr;
01329 T acc = prev;
01330
01331 for (i = 1; i < len; ++i)
01332 {
01333 T curr = arr[i];
01334 if (curr == prev + 1)
01335 {
01336 ++acc;
01337 prev = curr;
01338 }
01339 else
01340 {
01341 *pcurr++ = acc;
01342 acc = curr;
01343 *pcurr++ = curr-1;
01344 }
01345 prev = curr;
01346 }
01347 *pcurr = acc;
01348 if (acc != bm::gap_max_bits - 1)
01349 {
01350 ++pcurr;
01351 *pcurr = bm::gap_max_bits - 1;
01352 }
01353
01354 unsigned end = pcurr - buf;
01355
01356 *buf = (*buf & 7) + (end << 3);
01357 return end+1;
01358 }
01359
01360
01361
01362
01363
01364
01365
01366
01367
01368
01369
01370 template<typename T>
01371 unsigned bit_array_compute_gaps(const T* arr,
01372 unsigned len)
01373 {
01374 unsigned gap_count = 1;
01375 T prev = arr[0];
01376 if (prev > 0)
01377 ++gap_count;
01378 for (unsigned i = 1; i < len; ++i)
01379 {
01380 T curr = arr[i];
01381 if (curr != prev + 1)
01382 {
01383 gap_count += 2;
01384 }
01385 prev = curr;
01386 }
01387 return gap_count;
01388 }
01389
01390
01391
01392
01393
01394
01395
01396
01397
01398
01399
01400
01401 template<typename T> int gap_find_in_block(const T* buf,
01402 unsigned nbit,
01403 bm::id_t* prev)
01404 {
01405 BM_ASSERT(nbit < bm::gap_max_bits);
01406
01407 unsigned bitval;
01408 unsigned gap_idx = bm::gap_bfind(buf, nbit, &bitval);
01409
01410 if (bitval)
01411 {
01412 return 1;
01413 }
01414
01415 register unsigned val = buf[gap_idx] + 1;
01416 *prev += val - nbit;
01417
01418 return (val != bm::gap_max_bits);
01419 }
01420
01421
01422
01423
01424
01425
01426 BMFORCEINLINE void set_bit(unsigned* dest, unsigned bitpos)
01427 {
01428 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
01429 unsigned nword = unsigned(nbit >> bm::set_word_shift);
01430 nbit &= bm::set_word_mask;
01431 dest[nword] |= unsigned(1 << nbit);
01432 }
01433
01434
01435
01436
01437
01438
01439 BMFORCEINLINE unsigned test_bit(const unsigned* block, unsigned bitpos)
01440 {
01441 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
01442 unsigned nword = unsigned(nbit >> bm::set_word_shift);
01443 nbit &= bm::set_word_mask;
01444 return block[nword] & unsigned(1 << nbit);
01445 }
01446
01447
01448
01449
01450
01451
01452
01453
01454
01455
01456 inline void or_bit_block(unsigned* dest,
01457 unsigned bitpos,
01458 unsigned bitcount)
01459 {
01460 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
01461 unsigned nword = unsigned(nbit >> bm::set_word_shift);
01462 nbit &= bm::set_word_mask;
01463
01464 bm::word_t* word = dest + nword;
01465
01466 if (bitcount == 1)
01467 {
01468 *word |= unsigned(1 << nbit);
01469 return;
01470 }
01471
01472 if (nbit)
01473 {
01474 unsigned right_margin = nbit + bitcount;
01475
01476
01477
01478
01479 if (right_margin < 32)
01480 {
01481 unsigned mask =
01482 block_set_table<true>::_right[nbit] &
01483 block_set_table<true>::_left[right_margin-1];
01484 *word |= mask;
01485 return;
01486 }
01487 else
01488 {
01489 *word |= block_set_table<true>::_right[nbit];
01490 bitcount -= 32 - nbit;
01491 }
01492 ++word;
01493 }
01494
01495
01496
01497
01498 for ( ;bitcount >= 32; bitcount -= 32)
01499 {
01500 *word++ = 0xffffffff;
01501 }
01502
01503 if (bitcount)
01504 {
01505 *word |= block_set_table<true>::_left[bitcount-1];
01506 }
01507 }
01508
01509
01510
01511
01512
01513
01514
01515
01516
01517
01518 inline void sub_bit_block(unsigned* dest,
01519 unsigned bitpos,
01520 unsigned bitcount)
01521 {
01522 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
01523 unsigned nword = unsigned(nbit >> bm::set_word_shift);
01524 nbit &= bm::set_word_mask;
01525
01526 bm::word_t* word = dest + nword;
01527
01528 if (bitcount == 1)
01529 {
01530 *word &= ~unsigned(1 << nbit);
01531 return;
01532 }
01533
01534 if (nbit)
01535 {
01536 unsigned right_margin = nbit + bitcount;
01537
01538
01539
01540
01541 if (right_margin < 32)
01542 {
01543 unsigned mask =
01544 block_set_table<true>::_right[nbit] &
01545 block_set_table<true>::_left[right_margin-1];
01546 *word &= ~mask;
01547 return;
01548 }
01549 else
01550 {
01551 *word &= ~block_set_table<true>::_right[nbit];
01552 bitcount -= 32 - nbit;
01553 }
01554 ++word;
01555 }
01556
01557
01558
01559
01560 for ( ;bitcount >= 32; bitcount -= 32)
01561 {
01562 *word++ = 0;
01563 }
01564
01565 if (bitcount)
01566 {
01567 *word &= ~block_set_table<true>::_left[bitcount-1];
01568 }
01569 }
01570
01571
01572
01573
01574
01575
01576
01577
01578
01579
01580 inline void xor_bit_block(unsigned* dest,
01581 unsigned bitpos,
01582 unsigned bitcount)
01583 {
01584 unsigned nbit = unsigned(bitpos & bm::set_block_mask);
01585 unsigned nword = unsigned(nbit >> bm::set_word_shift);
01586 nbit &= bm::set_word_mask;
01587
01588 bm::word_t* word = dest + nword;
01589
01590 if (bitcount == 1)
01591 {
01592 *word ^= unsigned(1 << nbit);
01593 return;
01594 }
01595
01596 if (nbit)
01597 {
01598 unsigned right_margin = nbit + bitcount;
01599
01600
01601
01602
01603 if (right_margin < 32)
01604 {
01605 unsigned mask =
01606 block_set_table<true>::_right[nbit] &
01607 block_set_table<true>::_left[right_margin-1];
01608 *word ^= mask;
01609 return;
01610 }
01611 else
01612 {
01613 *word ^= block_set_table<true>::_right[nbit];
01614 bitcount -= 32 - nbit;
01615 }
01616 ++word;
01617 }
01618
01619
01620
01621
01622 for ( ;bitcount >= 32; bitcount -= 32)
01623 {
01624 *word++ ^= 0xffffffff;
01625 }
01626
01627 if (bitcount)
01628 {
01629 *word ^= block_set_table<true>::_left[bitcount-1];
01630 }
01631 }
01632
01633
01634
01635
01636
01637
01638
01639
01640
01641 template<typename T>
01642 void gap_sub_to_bitset(unsigned* dest, const T* buf)
01643 {
01644 register const T* pcurr = buf;
01645 register const T* pend = pcurr + (*pcurr >> 3);
01646 ++pcurr;
01647
01648 if (*buf & 1)
01649 {
01650 sub_bit_block(dest, 0, *pcurr + 1);
01651 ++pcurr;
01652 }
01653 ++pcurr;
01654
01655 while (pcurr <= pend)
01656 {
01657 unsigned bitpos = *(pcurr-1) + 1;
01658 BM_ASSERT(*pcurr > *(pcurr-1));
01659 unsigned gap_len = *pcurr - *(pcurr-1);
01660 sub_bit_block(dest, bitpos, gap_len);
01661 pcurr += 2;
01662 }
01663 }
01664
01665
01666
01667
01668
01669
01670
01671
01672
01673 template<typename T>
01674 void gap_xor_to_bitset(unsigned* dest, const T* buf)
01675 {
01676 register const T* pcurr = buf;
01677 register const T* pend = pcurr + (*pcurr >> 3);
01678 ++pcurr;
01679
01680 if (*buf & 1)
01681 {
01682 xor_bit_block(dest, 0, *pcurr + 1);
01683 ++pcurr;
01684 }
01685 ++pcurr;
01686
01687 while (pcurr <= pend)
01688 {
01689 unsigned bitpos = *(pcurr-1) + 1;
01690 BM_ASSERT(*pcurr > *(pcurr-1));
01691 unsigned gap_len = *pcurr - *(pcurr-1);
01692 xor_bit_block(dest, bitpos, gap_len);
01693 pcurr += 2;
01694 }
01695 }
01696
01697
01698
01699
01700
01701
01702
01703
01704
01705 template<typename T>
01706 void gap_add_to_bitset(unsigned* dest, const T* buf)
01707 {
01708 register const T* pcurr = buf;
01709 register const T* pend = pcurr + (*pcurr >> 3);
01710 ++pcurr;
01711
01712 if (*buf & 1)
01713 {
01714 or_bit_block(dest, 0, *pcurr + 1);
01715 ++pcurr;
01716 }
01717 ++pcurr;
01718
01719 while (pcurr <= pend)
01720 {
01721 unsigned bitpos = *(pcurr-1) + 1;
01722 BM_ASSERT(*pcurr > *(pcurr-1));
01723 unsigned gap_len = *pcurr - *(pcurr-1);
01724 or_bit_block(dest, bitpos, gap_len);
01725 pcurr += 2;
01726 }
01727 }
01728
01729
01730
01731
01732
01733
01734
01735
01736
01737 template<typename T>
01738 void gap_and_to_bitset(unsigned* dest, const T* buf)
01739 {
01740 register const T* pcurr = buf;
01741 register const T* pend = pcurr + (*pcurr >> 3);
01742 ++pcurr;
01743
01744 if (! (*buf & 1) )
01745 {
01746
01747 sub_bit_block(dest, 0, *pcurr + 1);
01748 ++pcurr;
01749 }
01750 ++pcurr;
01751
01752 while (pcurr <= pend)
01753 {
01754 unsigned bitpos = *(pcurr-1) + 1;
01755 BM_ASSERT(*pcurr > *(pcurr-1));
01756 unsigned gap_len = *pcurr - *(pcurr-1);
01757 sub_bit_block(dest, bitpos, gap_len);
01758 pcurr += 2;
01759 }
01760 }
01761
01762
01763
01764
01765
01766
01767
01768
01769
01770 template<typename T>
01771 bm::id_t gap_bitset_and_count(const unsigned* block, const T* buf)
01772 {
01773 BM_ASSERT(block);
01774
01775 register const T* pcurr = buf;
01776 register const T* pend = pcurr + (*pcurr >> 3);
01777 ++pcurr;
01778
01779 bm::id_t count = 0;
01780
01781 if (*buf & 1)
01782 {
01783 count += bit_block_calc_count_range(block, 0, *pcurr);
01784 ++pcurr;
01785 }
01786 ++pcurr;
01787
01788 while (pcurr <= pend)
01789 {
01790 bm::id_t c = bit_block_calc_count_range(block, *(pcurr-1)+1, *pcurr);
01791
01792 count += c;
01793 pcurr += 2;
01794 }
01795 return count;
01796 }
01797
01798
01799
01800
01801
01802
01803
01804
01805
01806 template<typename T>
01807 bm::id_t gap_bitset_and_any(const unsigned* block, const T* buf)
01808 {
01809 BM_ASSERT(block);
01810
01811 register const T* pcurr = buf;
01812 register const T* pend = pcurr + (*pcurr >> 3);
01813 ++pcurr;
01814
01815 bm::id_t count = 0;
01816 if (*buf & 1)
01817 {
01818 count += bit_block_any_range(block, 0, *pcurr);
01819 if (count)
01820 return count;
01821 ++pcurr;
01822 }
01823 ++pcurr;
01824
01825 while (pcurr <= pend)
01826 {
01827 bm::id_t c = bit_block_any_range(block, *(pcurr-1)+1, *pcurr);
01828 count += c;
01829 if (count)
01830 break;
01831 pcurr += 2;
01832 }
01833 return count;
01834 }
01835
01836
01837
01838
01839
01840
01841
01842
01843
01844
01845 template<typename T>
01846 bm::id_t gap_bitset_sub_count(const unsigned* block, const T* buf)
01847 {
01848 BM_ASSERT(block);
01849
01850 register const T* pcurr = buf;
01851 register const T* pend = pcurr + (*pcurr >> 3);
01852 ++pcurr;
01853
01854 bm::id_t count = 0;
01855
01856 if (!(*buf & 1))
01857 {
01858 count += bit_block_calc_count_range(block, 0, *pcurr);
01859 ++pcurr;
01860 }
01861 ++pcurr;
01862
01863 for (;pcurr <= pend; pcurr+=2)
01864 {
01865 count += bit_block_calc_count_range(block, *(pcurr-1)+1, *pcurr);
01866 }
01867 return count;
01868 }
01869
01870
01871
01872
01873
01874
01875
01876
01877
01878 template<typename T>
01879 bm::id_t gap_bitset_sub_any(const unsigned* block, const T* buf)
01880 {
01881 BM_ASSERT(block);
01882
01883 register const T* pcurr = buf;
01884 register const T* pend = pcurr + (*pcurr >> 3);
01885 ++pcurr;
01886
01887 bm::id_t count = 0;
01888
01889 if (!(*buf & 1))
01890 {
01891 count += bit_block_any_range(block, 0, *pcurr);
01892 if (count)
01893 return count;
01894 ++pcurr;
01895 }
01896 ++pcurr;
01897
01898 for (;pcurr <= pend; pcurr+=2)
01899 {
01900 count += bit_block_any_range(block, *(pcurr-1)+1, *pcurr);
01901 if (count)
01902 return count;
01903 }
01904 return count;
01905 }
01906
01907
01908
01909
01910
01911
01912
01913
01914
01915
01916 template<typename T>
01917 bm::id_t gap_bitset_xor_count(const unsigned* block, const T* buf)
01918 {
01919 BM_ASSERT(block);
01920
01921 register const T* pcurr = buf;
01922 register const T* pend = pcurr + (*pcurr >> 3);
01923 ++pcurr;
01924
01925 unsigned bitval = *buf & 1;
01926
01927 register bm::id_t count = bit_block_calc_count_range(block, 0, *pcurr);
01928 if (bitval)
01929 {
01930 count = *pcurr + 1 - count;
01931 }
01932
01933 for (bitval^=1, ++pcurr; pcurr <= pend; bitval^=1, ++pcurr)
01934 {
01935 T prev = *(pcurr-1)+1;
01936 bm::id_t c = bit_block_calc_count_range(block, prev, *pcurr);
01937
01938 if (bitval)
01939 {
01940 c = (*pcurr - prev + 1) - c;
01941 }
01942 count += c;
01943 }
01944 return count;
01945 }
01946
01947
01948
01949
01950
01951
01952
01953
01954 template<typename T>
01955 bm::id_t gap_bitset_xor_any(const unsigned* block, const T* buf)
01956 {
01957 BM_ASSERT(block);
01958
01959 register const T* pcurr = buf;
01960 register const T* pend = pcurr + (*pcurr >> 3);
01961 ++pcurr;
01962
01963 unsigned bitval = *buf & 1;
01964
01965 register bm::id_t count = bit_block_any_range(block, 0, *pcurr);
01966 if (bitval)
01967 {
01968 count = *pcurr + 1 - count;
01969 }
01970
01971 for (bitval^=1, ++pcurr; pcurr <= pend; bitval^=1, ++pcurr)
01972 {
01973 T prev = *(pcurr-1)+1;
01974 bm::id_t c = bit_block_any_range(block, prev, *pcurr);
01975
01976 if (bitval)
01977 {
01978 c = (*pcurr - prev + 1) - c;
01979 }
01980
01981 count += c;
01982 if (count)
01983 return count;
01984 }
01985 return count;
01986 }
01987
01988
01989
01990
01991
01992
01993
01994
01995
01996
01997 template<typename T>
01998 bm::id_t gap_bitset_or_count(const unsigned* block, const T* buf)
01999 {
02000 BM_ASSERT(block);
02001
02002 register const T* pcurr = buf;
02003 register const T* pend = pcurr + (*pcurr >> 3);
02004 ++pcurr;
02005
02006 unsigned bitval = *buf & 1;
02007
02008 register bm::id_t count;
02009 if (bitval)
02010 {
02011 count = *pcurr + 1;
02012 }
02013 else
02014 {
02015 count = bit_block_calc_count_range(block, 0, *pcurr);
02016 }
02017
02018 for (bitval^=1, ++pcurr; pcurr <= pend; bitval^=1, ++pcurr)
02019 {
02020 T prev = *(pcurr-1)+1;
02021 bm::id_t c;
02022
02023 if (bitval)
02024 {
02025 c = (*pcurr - prev + 1);
02026 }
02027 else
02028 {
02029 c = bit_block_calc_count_range(block, prev, *pcurr);
02030 }
02031
02032 count += c;
02033 }
02034 return count;
02035 }
02036
02037
02038
02039
02040
02041
02042
02043
02044 template<typename T>
02045 bm::id_t gap_bitset_or_any(const unsigned* block, const T* buf)
02046 {
02047 BM_ASSERT(block);
02048
02049 register const T* pcurr = buf;
02050 register const T* pend = pcurr + (*pcurr >> 3);
02051 ++pcurr;
02052
02053 unsigned bitval = *buf & 1;
02054
02055 register bm::id_t count;
02056 if (bitval)
02057 {
02058 count = *pcurr + 1;
02059 }
02060 else
02061 {
02062 count = bit_block_any_range(block, 0, *pcurr);
02063 }
02064
02065 for (bitval^=1, ++pcurr; pcurr <= pend; bitval^=1, ++pcurr)
02066 {
02067 T prev = *(pcurr-1)+1;
02068 bm::id_t c;
02069
02070 if (bitval)
02071 {
02072 c = (*pcurr - prev + 1);
02073 }
02074 else
02075 {
02076 c = bit_block_any_range(block, prev, *pcurr);
02077 }
02078 count += c;
02079 if (count)
02080 return count;
02081 }
02082 return count;
02083 }
02084
02085
02086
02087
02088
02089
02090
02091
02092
02093
02094
02095 inline
02096 void bit_block_set(bm::word_t* BMRESTRICT dst, bm::word_t value)
02097 {
02098
02099
02100
02101 ::memset(dst, value, bm::set_block_size * sizeof(bm::word_t));
02102
02103 }
02104
02105
02106
02107
02108
02109
02110
02111
02112
02113 template<typename T>
02114 void gap_convert_to_bitset(unsigned* dest, const T* buf)
02115 {
02116 bit_block_set(dest, 0);
02117 gap_add_to_bitset(dest, buf);
02118 }
02119
02120
02121
02122
02123
02124
02125
02126
02127
02128
02129 template<typename T>
02130 void gap_convert_to_bitset(unsigned* dest, const T* buf, unsigned dest_len)
02131 {
02132 ::memset(dest, 0, dest_len * sizeof(unsigned));
02133 gap_add_to_bitset(dest, buf);
02134 }
02135
02136
02137
02138
02139
02140
02141
02142
02143
02144
02145
02146
02147
02148
02149
02150 template<typename T>
02151 unsigned* gap_convert_to_bitset_smart(unsigned* dest,
02152 const T* buf,
02153 id_t set_max)
02154 {
02155 if (buf[1] == set_max - 1)
02156 {
02157 return (buf[0] & 1) ? FULL_BLOCK_ADDR : 0;
02158 }
02159
02160 gap_convert_to_bitset(dest, buf);
02161 return dest;
02162 }
02163
02164
02165
02166
02167
02168
02169
02170
02171
02172
02173 template<typename T> unsigned gap_control_sum(const T* buf)
02174 {
02175 unsigned end = *buf >> 3;
02176
02177 register const T* pcurr = buf;
02178 register const T* pend = pcurr + (*pcurr >> 3);
02179 ++pcurr;
02180
02181 if (*buf & 1)
02182 {
02183 ++pcurr;
02184 }
02185 ++pcurr;
02186
02187 while (pcurr <= pend)
02188 {
02189 BM_ASSERT(*pcurr > *(pcurr-1));
02190 pcurr += 2;
02191 }
02192 return buf[end];
02193
02194 }
02195
02196
02197
02198
02199
02200
02201
02202
02203
02204 template<class T> void gap_set_all(T* buf,
02205 unsigned set_max,
02206 unsigned value)
02207 {
02208 BM_ASSERT(value == 0 || value == 1);
02209 *buf = (*buf & 6u) + (1u << 3) + value;
02210 *(++buf) = set_max - 1;
02211 }
02212
02213
02214
02215
02216
02217
02218
02219
02220
02221
02222
02223
02224 template<class T>
02225 void gap_init_range_block(T* buf,
02226 unsigned from,
02227 unsigned to,
02228 unsigned value,
02229 unsigned set_max)
02230 {
02231 BM_ASSERT(value == 0 || value == 1);
02232
02233 unsigned gap_len;
02234 if (from == 0)
02235 {
02236 if (to == set_max - 1)
02237 {
02238 gap_set_all(buf, set_max, value);
02239 }
02240 else
02241 {
02242 gap_len = 2;
02243 buf[1] = to;
02244 buf[2] = set_max - 1;
02245 buf[0] = (*buf & 6u) + (gap_len << 3) + value;
02246 }
02247 return;
02248 }
02249
02250
02251 value = !value;
02252 if (to == set_max - 1)
02253 {
02254 gap_len = 2;
02255 buf[1] = from - 1;
02256 buf[2] = set_max - 1;
02257 }
02258 else
02259 {
02260 gap_len = 3;
02261 buf[1] = from - 1;
02262 buf[2] = to;
02263 buf[3] = set_max - 1;
02264 }
02265 buf[0] = (*buf & 6u) + (gap_len << 3) + value;
02266 }
02267
02268
02269
02270
02271
02272
02273
02274
02275 template<typename T> void gap_invert(T* buf)
02276 {
02277 *buf ^= 1;
02278 }
02279
02280
02281
02282
02283
02284
02285
02286
02287
02288
02289
02290
02291
02292
02293
02294
02295
02296
02297
02298
02299
02300
02301
02302
02303
02304
02305
02306 template<typename T>
02307 bool gap_is_all_zero(const T* buf, unsigned set_max)
02308 {
02309 return (((*buf & 1)==0) && (*(++buf) == set_max - 1));
02310 }
02311
02312
02313
02314
02315
02316
02317
02318
02319
02320 template<typename T>
02321 bool gap_is_all_one(const T* buf, unsigned set_max)
02322 {
02323 return ((*buf & 1) && (*(++buf) == set_max - 1));
02324 }
02325
02326
02327
02328
02329
02330
02331
02332
02333 template<typename T> unsigned gap_length(const T* buf)
02334 {
02335 return (*buf >> 3) + 1;
02336 }
02337
02338
02339
02340
02341
02342
02343
02344
02345
02346 template<typename T>
02347 unsigned gap_capacity(const T* buf, const T* glevel_len)
02348 {
02349 return glevel_len[(*buf >> 1) & 3];
02350 }
02351
02352
02353
02354
02355
02356
02357
02358
02359
02360
02361 template<typename T>
02362 unsigned gap_limit(const T* buf, const T* glevel_len)
02363 {
02364 return glevel_len[(*buf >> 1) & 3]-4;
02365 }
02366
02367
02368
02369
02370
02371
02372
02373
02374
02375 template<typename T> unsigned gap_level(const T* buf)
02376 {
02377 return (*buf >> 1) & 3;
02378 }
02379
02380
02381
02382
02383
02384
02385
02386
02387
02388 template<typename T>
02389 void set_gap_level(T* buf, unsigned level)
02390 {
02391 BM_ASSERT(level < bm::gap_levels);
02392 *buf = ((level & 3) << 1) | (*buf & 1) | (*buf & ~7);
02393 }
02394
02395
02396
02397
02398
02399
02400
02401
02402
02403
02404 template<typename T>
02405 inline int gap_calc_level(int len, const T* glevel_len)
02406 {
02407 if (len <= (glevel_len[0]-4)) return 0;
02408 if (len <= (glevel_len[1]-4)) return 1;
02409 if (len <= (glevel_len[2]-4)) return 2;
02410 if (len <= (glevel_len[3]-4)) return 3;
02411
02412 BM_ASSERT(bm::gap_levels == 4);
02413 return -1;
02414 }
02415
02416
02417
02418
02419
02420
02421
02422
02423
02424
02425 template<typename T>
02426 inline unsigned gap_free_elements(const T* buf, const T* glevel_len)
02427 {
02428 unsigned len = gap_length(buf);
02429 unsigned capacity = gap_capacity(buf, glevel_len);
02430 return capacity - len;
02431 }
02432
02433
02434
02435
02436
02437
02438
02439
02440
02441
02442
02443 template<typename T>
02444 int bitcmp(const T* buf1, const T* buf2, unsigned len)
02445 {
02446 BM_ASSERT(len);
02447 const T* pend1 = buf1 + len;
02448 do
02449 {
02450 T w1 = *buf1++;
02451 T w2 = *buf2++;
02452 T diff = w1 ^ w2;
02453
02454 if (diff)
02455 {
02456 return (w1 & diff & -diff) ? 1 : -1;
02457 }
02458
02459 } while (buf1 < pend1);
02460
02461 return 0;
02462 }
02463
02464
02465
02466
02467
02468
02469
02470
02471
02472
02473
02474
02475
02476 template<typename T>
02477 unsigned bit_convert_to_gap(T* BMRESTRICT dest,
02478 const unsigned* BMRESTRICT src,
02479 bm::id_t bits,
02480 unsigned dest_len)
02481 {
02482 register T* BMRESTRICT pcurr = dest;
02483 T* BMRESTRICT end = dest + dest_len;
02484 register int bitval = (*src) & 1;
02485
02486 *pcurr = bitval;
02487
02488 ++pcurr;
02489 *pcurr = 0;
02490 register unsigned bit_idx = 0;
02491 register int bitval_next;
02492
02493 unsigned val = *src;
02494
02495 do
02496 {
02497
02498
02499 while (val == 0 || val == 0xffffffff)
02500 {
02501 bitval_next = val ? 1 : 0;
02502 if (bitval != bitval_next)
02503 {
02504 *pcurr++ = bit_idx-1;
02505 BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
02506 if (pcurr >= end)
02507 {
02508 return 0;
02509 }
02510 bitval = bitval_next;
02511 }
02512 bit_idx += sizeof(*src) * 8;
02513 if (bit_idx >= bits)
02514 {
02515 goto complete;
02516 }
02517 ++src;
02518 val = *src;
02519 }
02520
02521
02522 register unsigned mask = 1;
02523 while (mask)
02524 {
02525
02526
02527 bitval_next = val & mask ? 1 : 0;
02528 if (bitval != bitval_next)
02529 {
02530 *pcurr++ = bit_idx-1;
02531 BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
02532 bitval = bitval_next;
02533 if (pcurr >= end)
02534 {
02535 return 0;
02536 }
02537 }
02538
02539 mask <<= 1;
02540 ++bit_idx;
02541
02542 }
02543
02544 if (bit_idx >= bits)
02545 {
02546 goto complete;
02547 }
02548
02549 ++src;
02550 val = *src;
02551
02552 } while(1);
02553
02554 complete:
02555 *pcurr = bit_idx-1;
02556 unsigned len = (unsigned)(pcurr - dest);
02557 *dest = (*dest & 7) + (len << 3);
02558 return len;
02559 }
02560
02561
02562
02563
02564
02565
02566 template<class T, class F>
02567 void for_each_gap_dbit(const T* buf, F& func)
02568 {
02569 const T* pcurr = buf;
02570 const T* pend = pcurr + (*pcurr >> 3);
02571
02572 ++pcurr;
02573
02574 unsigned prev = 0;
02575 unsigned first_inc;
02576
02577 if (*buf & 1)
02578 {
02579 first_inc = 0;
02580 unsigned to = *pcurr;
02581 for (unsigned i = 0; i <= to; ++i)
02582 {
02583 func(1);
02584 }
02585 prev = to;
02586 ++pcurr;
02587 }
02588 else
02589 {
02590 first_inc = 1;
02591 }
02592 ++pcurr;
02593
02594 while (pcurr <= pend)
02595 {
02596 unsigned from = *(pcurr-1)+1;
02597 unsigned to = *pcurr;
02598 if (first_inc)
02599 {
02600 func(from - prev + first_inc);
02601 first_inc = 0;
02602 }
02603 else
02604 {
02605 func(from - prev);
02606 }
02607
02608 for (unsigned i = from+1; i <= to; ++i)
02609 {
02610 func(1);
02611 }
02612 prev = to;
02613 pcurr += 2;
02614 }
02615 }
02616
02617
02618
02619
02620
02621 template<typename D, typename T>
02622 D gap_convert_to_arr(D* BMRESTRICT dest,
02623 const T* BMRESTRICT buf,
02624 unsigned dest_len,
02625 bool invert = false)
02626 {
02627 register const T* BMRESTRICT pcurr = buf;
02628 register const T* pend = pcurr + (*pcurr >> 3);
02629
02630 D* BMRESTRICT dest_curr = dest;
02631 ++pcurr;
02632
02633 int bitval = (*buf) & 1;
02634 if (invert)
02635 bitval = !bitval;
02636
02637 if (bitval)
02638 {
02639 if (unsigned(*pcurr + 1) >= dest_len)
02640 return 0;
02641 dest_len -= *pcurr;
02642 T to = *pcurr;
02643 for (T i = 0; ;++i)
02644 {
02645 *dest_curr++ = i;
02646 if (i == to) break;
02647 }
02648 ++pcurr;
02649 }
02650 ++pcurr;
02651
02652 while (pcurr <= pend)
02653 {
02654 unsigned pending = *pcurr - *(pcurr-1);
02655 if (pending >= dest_len)
02656 return 0;
02657 dest_len -= pending;
02658 T from = *(pcurr-1)+1;
02659 T to = *pcurr;
02660 for (T i = from; ;++i)
02661 {
02662 *dest_curr++ = i;
02663 if (i == to) break;
02664 }
02665 pcurr += 2;
02666 }
02667 return (D) (dest_curr - dest);
02668 }
02669
02670
02671
02672
02673
02674
02675
02676
02677
02678
02679
02680 inline
02681 bm::id_t bit_block_calc_count(const bm::word_t* block,
02682 const bm::word_t* block_end)
02683 {
02684 BM_ASSERT(block < block_end);
02685 bm::id_t count = 0;
02686
02687 #ifdef BMVECTOPT
02688 count = VECT_BITCOUNT(block, block_end);
02689 #else
02690 #ifdef BM64OPT
02691
02692
02693
02694 const bm::id64_t* b1 = (bm::id64_t*) block;
02695 const bm::id64_t* b2 = (bm::id64_t*) block_end;
02696 do
02697 {
02698 count += bitcount64_4way(b1[0], b1[1], b1[2], b1[3]);
02699 b1 += 4;
02700 } while (b1 < b2);
02701 #else
02702
02703
02704
02705
02706 bm::word_t acc = *block++;
02707 do
02708 {
02709 bm::word_t in = *block++;
02710 bm::word_t acc_prev = acc;
02711 acc |= in;
02712 if (acc_prev &= in)
02713 {
02714 BM_INCWORD_BITCOUNT(count, acc);
02715 acc = acc_prev;
02716 }
02717 } while (block < block_end);
02718
02719 BM_INCWORD_BITCOUNT(count, acc);
02720
02721 #endif
02722 #endif
02723 return count;
02724 }
02725
02726
02727
02728
02729
02730
02731
02732
02733
02734
02735
02736
02737
02738
02739
02740 inline
02741 bm::id_t bit_count_change(bm::word_t w)
02742 {
02743 unsigned count = 1;
02744 w ^= (w >> 1);
02745
02746 BM_INCWORD_BITCOUNT(count, w);
02747 count -= (w >> ((sizeof(w) * 8) - 1));
02748 return count;
02749 }
02750
02751
02752
02753
02754
02755
02756 inline
02757 void bit_count_change32(const bm::word_t* block,
02758 const bm::word_t* block_end,
02759 unsigned* bit_count,
02760 unsigned* gap_count)
02761 {
02762 BM_ASSERT(block < block_end);
02763 BM_ASSERT(bit_count);
02764 BM_ASSERT(gap_count);
02765
02766 *gap_count = 1;
02767 *bit_count = 0;
02768
02769 bm::word_t w, w0, w_prev, w_l;
02770 w = w0 = *block;
02771
02772 BM_INCWORD_BITCOUNT(*bit_count, w);
02773
02774 const int w_shift = sizeof(w) * 8 - 1;
02775 w ^= (w >> 1);
02776 BM_INCWORD_BITCOUNT(*gap_count, w);
02777 *gap_count -= (w_prev = (w0 >> w_shift));
02778
02779 for (++block ;block < block_end; ++block)
02780 {
02781 w = w0 = *block;
02782 ++(*gap_count);
02783
02784 if (!w)
02785 {
02786 *gap_count -= !w_prev;
02787 w_prev = 0;
02788 }
02789 else
02790 {
02791 BM_INCWORD_BITCOUNT(*bit_count, w);
02792
02793 w ^= (w >> 1);
02794 BM_INCWORD_BITCOUNT(*gap_count, w);
02795
02796 w_l = w0 & 1;
02797 *gap_count -= (w0 >> w_shift);
02798 *gap_count -= !(w_prev ^ w_l);
02799
02800 w_prev = (w0 >> w_shift);
02801 }
02802 }
02803
02804 }
02805
02806
02807
02808
02809
02810
02811
02812
02813
02814
02815
02816
02817
02818 inline
02819 bm::id_t bit_block_calc_count_change(const bm::word_t* block,
02820 const bm::word_t* block_end,
02821 unsigned* bit_count)
02822 {
02823 #if defined(BMSSE2OPT) || defined(BMSSE42OPT)
02824
02825 #ifdef BMSSE42OPT
02826 return sse4_bit_block_calc_count_change(
02827 (const __m128i*)block, (const __m128i*)block_end, bit_count);
02828 #else
02829 # ifdef BMSSE2OPT
02830 return sse2_bit_block_calc_count_change(
02831 (const __m128i*)block, (const __m128i*)block_end, bit_count);
02832 # endif
02833 #endif
02834
02835 #else // non-SSE code
02836
02837 BM_ASSERT(block < block_end);
02838 BM_ASSERT(bit_count);
02839
02840
02841 #ifdef BM64OPT
02842 bm::id_t count = 1;
02843 *bit_count = 0;
02844
02845
02846
02847 const bm::id64_t* b1 = (bm::id64_t*) block;
02848 const bm::id64_t* b2 = (bm::id64_t*) block_end;
02849
02850 bm::id64_t w, w0, w_prev, w_l;
02851 w = w0 = *b1;
02852
02853 *bit_count = word_bitcount64(w);
02854
02855 const int w_shift = sizeof(w) * 8 - 1;
02856 w ^= (w >> 1);
02857 count += word_bitcount64(w);
02858 count -= (w_prev = (w0 >> w_shift));
02859
02860
02861 for (++b1 ;b1 < b2; ++b1)
02862 {
02863 w = w0 = *b1;
02864
02865 ++count;
02866
02867 if (!w)
02868 {
02869 count -= !w_prev;
02870 w_prev = 0;
02871 }
02872 else
02873 {
02874 *bit_count += word_bitcount64(w);
02875 w ^= (w >> 1);
02876 count += word_bitcount64(w);
02877
02878 w_l = w0 & 1;
02879 count -= (w0 >> w_shift);
02880 count -= !(w_prev ^ w_l);
02881
02882 w_prev = (w0 >> w_shift);
02883 }
02884 }
02885 return count;
02886
02887 #else
02888 unsigned gap_count;
02889 bit_count_change32(block, block_end, bit_count, &gap_count);
02890 return gap_count;
02891 #endif
02892
02893 #endif
02894 }
02895
02896
02897
02898
02899
02900
02901
02902
02903
02904 inline
02905 bm::id_t bit_block_calc_count_range(const bm::word_t* block,
02906 bm::word_t left,
02907 bm::word_t right)
02908 {
02909 BM_ASSERT(left <= right);
02910
02911 unsigned nbit = left;
02912 unsigned nword = unsigned(nbit >> bm::set_word_shift);
02913 nbit &= bm::set_word_mask;
02914
02915 const bm::word_t* word = block + nword;
02916
02917 if (left == right)
02918 {
02919 return (*word >> nbit) & 1;
02920 }
02921 bm::id_t count = 0;
02922
02923 unsigned acc;
02924 unsigned bitcount = right - left + 1;
02925
02926 if (nbit)
02927 {
02928 unsigned right_margin = nbit + (right - left);
02929
02930 if (right_margin < 32)
02931 {
02932 unsigned mask =
02933 block_set_table<true>::_right[nbit] &
02934 block_set_table<true>::_left[right_margin];
02935 acc = *word & mask;
02936
02937 BM_INCWORD_BITCOUNT(count, acc);
02938 return count;
02939 }
02940 else
02941 {
02942 acc = *word & block_set_table<true>::_right[nbit];
02943 BM_INCWORD_BITCOUNT(count, acc);
02944 bitcount -= 32 - nbit;
02945 }
02946 ++word;
02947 }
02948
02949
02950 for ( ;bitcount >= 32; bitcount -= 32)
02951 {
02952 acc = *word++;
02953 BM_INCWORD_BITCOUNT(count, acc);
02954 }
02955
02956 if (bitcount)
02957 {
02958 acc = (*word) & block_set_table<true>::_left[bitcount-1];
02959 BM_INCWORD_BITCOUNT(count, acc);
02960 }
02961
02962 return count;
02963 }
02964
02965
02966
02967
02968
02969
02970
02971
02972
02973 inline
02974 bm::id_t bit_block_any_range(const bm::word_t* block,
02975 bm::word_t left,
02976 bm::word_t right)
02977 {
02978 BM_ASSERT(left <= right);
02979
02980 unsigned nbit = left;
02981 unsigned nword = unsigned(nbit >> bm::set_word_shift);
02982 nbit &= bm::set_word_mask;
02983
02984 const bm::word_t* word = block + nword;
02985
02986 if (left == right)
02987 {
02988 return (*word >> nbit) & 1;
02989 }
02990 unsigned acc;
02991 unsigned bitcount = right - left + 1;
02992
02993 if (nbit)
02994 {
02995 unsigned right_margin = nbit + (right - left);
02996 if (right_margin < 32)
02997 {
02998 unsigned mask =
02999 block_set_table<true>::_right[nbit] &
03000 block_set_table<true>::_left[right_margin];
03001 acc = *word & mask;
03002 return acc;
03003 }
03004 else
03005 {
03006 acc = *word & block_set_table<true>::_right[nbit];
03007 if (acc)
03008 return acc;
03009 bitcount -= 32 - nbit;
03010 }
03011 ++word;
03012 }
03013
03014
03015 for ( ;bitcount >= 32; bitcount -= 32)
03016 {
03017 acc = *word++;
03018 if (acc)
03019 return acc;
03020 }
03021
03022 if (bitcount)
03023 {
03024 acc = (*word) & block_set_table<true>::_left[bitcount-1];
03025 if (acc)
03026 return acc;
03027 }
03028
03029 return 0;
03030 }
03031
03032
03033
03034
03035
03036
03037
03038
03039 template<typename T> void bit_invert(T* start, T* end)
03040 {
03041 #ifdef BMVECTOPT
03042 VECT_INVERT_ARR(start, end);
03043 #else
03044 do
03045 {
03046 start[0] = ~start[0];
03047 start[1] = ~start[1];
03048 start[2] = ~start[2];
03049 start[3] = ~start[3];
03050 start+=4;
03051 } while (start < end);
03052 #endif
03053 }
03054
03055
03056
03057
03058
03059
03060 inline bool is_bits_one(const bm::wordop_t* start,
03061 const bm::wordop_t* end)
03062 {
03063 do
03064 {
03065 bm::wordop_t tmp =
03066 start[0] & start[1] & start[2] & start[3];
03067 if (tmp != bm::all_bits_mask)
03068 return false;
03069 start += 4;
03070 } while (start < end);
03071
03072 return true;
03073 }
03074
03075
03076
03077
03078
03079
03080
03081 inline bool bit_is_all_zero(const bm::wordop_t* start,
03082 const bm::wordop_t* end)
03083 {
03084 do
03085 {
03086 bm::wordop_t tmp =
03087 start[0] | start[1] | start[2] | start[3];
03088 if (tmp)
03089 return false;
03090 start += 4;
03091 } while (start < end);
03092
03093 return true;
03094 }
03095
03096
03097
03098
03099
03100
03101 BMFORCEINLINE unsigned and_op(unsigned v1, unsigned v2)
03102 {
03103 return v1 & v2;
03104 }
03105
03106
03107
03108 BMFORCEINLINE unsigned xor_op(unsigned v1, unsigned v2)
03109 {
03110 return v1 ^ v2;
03111 }
03112
03113
03114
03115
03116
03117
03118
03119
03120
03121
03122
03123
03124
03125
03126
03127
03128
03129
03130 inline gap_word_t* gap_operation_and(const gap_word_t* BMRESTRICT vect1,
03131 const gap_word_t* BMRESTRICT vect2,
03132 gap_word_t* BMRESTRICT tmp_buf,
03133 unsigned& dsize)
03134 {
03135 gap_buff_op(tmp_buf, vect1, 0, vect2, 0, and_op, dsize);
03136 return tmp_buf;
03137 }
03138
03139
03140
03141
03142
03143
03144
03145
03146
03147
03148
03149
03150
03151
03152
03153 inline unsigned gap_operation_any_and(const gap_word_t* BMRESTRICT vect1,
03154 const gap_word_t* BMRESTRICT vect2)
03155 {
03156 return gap_buff_any_op(vect1, 0, vect2, 0, and_op);
03157 }
03158
03159
03160
03161
03162
03163
03164
03165
03166
03167
03168
03169
03170
03171
03172
03173
03174
03175
03176
03177 inline gap_word_t* gap_operation_xor(const gap_word_t* BMRESTRICT vect1,
03178 const gap_word_t* BMRESTRICT vect2,
03179 gap_word_t* BMRESTRICT tmp_buf,
03180 unsigned& dsize)
03181 {
03182 gap_buff_op(tmp_buf, vect1, 0, vect2, 0, xor_op, dsize);
03183 return tmp_buf;
03184 }
03185
03186
03187
03188
03189
03190
03191
03192
03193
03194
03195
03196
03197
03198
03199
03200
03201 inline unsigned gap_operation_any_xor(const gap_word_t* BMRESTRICT vect1,
03202 const gap_word_t* BMRESTRICT vect2)
03203 {
03204 return gap_buff_any_op(vect1, 0, vect2, 0, xor_op);
03205 }
03206
03207
03208
03209
03210
03211
03212
03213
03214
03215
03216
03217
03218
03219
03220
03221
03222
03223
03224
03225
03226 inline gap_word_t* gap_operation_or(const gap_word_t* BMRESTRICT vect1,
03227 const gap_word_t* BMRESTRICT vect2,
03228 gap_word_t* BMRESTRICT tmp_buf,
03229 unsigned& dsize)
03230 {
03231 gap_buff_op(tmp_buf, vect1, 1, vect2, 1, and_op, dsize);
03232 gap_invert(tmp_buf);
03233 return tmp_buf;
03234 }
03235
03236
03237
03238
03239
03240
03241
03242
03243
03244
03245
03246
03247
03248
03249
03250
03251
03252
03253
03254
03255
03256 inline gap_word_t* gap_operation_sub(const gap_word_t* BMRESTRICT vect1,
03257 const gap_word_t* BMRESTRICT vect2,
03258 gap_word_t* BMRESTRICT tmp_buf,
03259 unsigned& dsize)
03260 {
03261 gap_buff_op(tmp_buf, vect1, 0, vect2, 1, and_op, dsize);
03262 return tmp_buf;
03263 }
03264
03265
03266
03267
03268
03269
03270
03271
03272
03273
03274
03275
03276
03277
03278
03279
03280 inline unsigned gap_operation_any_sub(const gap_word_t* BMRESTRICT vect1,
03281 const gap_word_t* BMRESTRICT vect2)
03282 {
03283 return gap_buff_any_op(vect1, 0, vect2, 1, and_op);
03284 }
03285
03286
03287
03288
03289
03290
03291
03292
03293
03294
03295
03296
03297
03298
03299
03300 inline
03301 void bit_block_copy(bm::word_t* BMRESTRICT dst, const bm::word_t* BMRESTRICT src)
03302 {
03303 #ifdef BMVECTOPT
03304 VECT_COPY_BLOCK(dst, src, src + bm::set_block_size);
03305 #else
03306 ::memcpy(dst, src, bm::set_block_size * sizeof(bm::word_t));
03307 #endif
03308 }
03309
03310
03311
03312
03313
03314
03315
03316
03317
03318
03319
03320 inline
03321 void bit_block_and(bm::word_t* BMRESTRICT dst, const bm::word_t* BMRESTRICT src)
03322 {
03323 #ifdef BMVECTOPT
03324 VECT_AND_ARR(dst, src, src + bm::set_block_size);
03325 #else
03326 const bm::wordop_t* BMRESTRICT wrd_ptr = (wordop_t*)src;
03327 const bm::wordop_t* BMRESTRICT wrd_end = (wordop_t*)(src + bm::set_block_size);
03328 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
03329
03330 do
03331 {
03332 dst_ptr[0] &= wrd_ptr[0];
03333 dst_ptr[1] &= wrd_ptr[1];
03334 dst_ptr[2] &= wrd_ptr[2];
03335 dst_ptr[3] &= wrd_ptr[3];
03336
03337 dst_ptr+=4;
03338 wrd_ptr+=4;
03339 } while (wrd_ptr < wrd_end);
03340 #endif
03341 }
03342
03343
03344
03345
03346
03347
03348
03349
03350
03351
03352
03353
03354 inline
03355 unsigned bit_block_and_count(const bm::word_t* src1,
03356 const bm::word_t* src1_end,
03357 const bm::word_t* src2)
03358 {
03359 unsigned count;
03360 #ifdef BMVECTOPT
03361 count = VECT_BITCOUNT_AND(src1, src1_end, src2);
03362 #else
03363 count = 0;
03364 # ifdef BM64OPT
03365 const bm::id64_t* b1 = (bm::id64_t*) src1;
03366 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
03367 const bm::id64_t* b2 = (bm::id64_t*) src2;
03368 do
03369 {
03370 count += bitcount64_4way(b1[0] & b2[0],
03371 b1[1] & b2[1],
03372 b1[2] & b2[2],
03373 b1[3] & b2[3]);
03374 b1 += 4;
03375 b2 += 4;
03376 } while (b1 < b1_end);
03377 # else
03378 do
03379 {
03380 BM_INCWORD_BITCOUNT(count, src1[0] & src2[0]);
03381 BM_INCWORD_BITCOUNT(count, src1[1] & src2[1]);
03382 BM_INCWORD_BITCOUNT(count, src1[2] & src2[2]);
03383 BM_INCWORD_BITCOUNT(count, src1[3] & src2[3]);
03384
03385 src1+=4;
03386 src2+=4;
03387 } while (src1 < src1_end);
03388 # endif
03389 #endif
03390 return count;
03391 }
03392
03393
03394
03395
03396
03397
03398
03399
03400
03401
03402
03403
03404 inline
03405 unsigned bit_block_and_any(const bm::word_t* src1,
03406 const bm::word_t* src1_end,
03407 const bm::word_t* src2)
03408 {
03409 unsigned count = 0;
03410 do
03411 {
03412 count = (src1[0] & src2[0]) |
03413 (src1[1] & src2[1]) |
03414 (src1[2] & src2[2]) |
03415 (src1[3] & src2[3]);
03416
03417 src1+=4; src2+=4;
03418 } while ((src1 < src1_end) && (count == 0));
03419 return count;
03420 }
03421
03422
03423
03424
03425
03426
03427
03428
03429
03430
03431
03432
03433
03434
03435 inline
03436 unsigned bit_block_xor_count(const bm::word_t* BMRESTRICT src1,
03437 const bm::word_t* BMRESTRICT src1_end,
03438 const bm::word_t* BMRESTRICT src2)
03439 {
03440 unsigned count;
03441 #ifdef BMVECTOPT
03442 count = VECT_BITCOUNT_XOR(src1, src1_end, src2);
03443 #else
03444 count = 0;
03445 # ifdef BM64OPT
03446 const bm::id64_t* b1 = (bm::id64_t*) src1;
03447 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
03448 const bm::id64_t* b2 = (bm::id64_t*) src2;
03449 do
03450 {
03451 count += bitcount64_4way(b1[0] ^ b2[0],
03452 b1[1] ^ b2[1],
03453 b1[2] ^ b2[2],
03454 b1[3] ^ b2[3]);
03455 b1 += 4;
03456 b2 += 4;
03457 } while (b1 < b1_end);
03458 # else
03459 do
03460 {
03461 BM_INCWORD_BITCOUNT(count, src1[0] ^ src2[0]);
03462 BM_INCWORD_BITCOUNT(count, src1[1] ^ src2[1]);
03463 BM_INCWORD_BITCOUNT(count, src1[2] ^ src2[2]);
03464 BM_INCWORD_BITCOUNT(count, src1[3] ^ src2[3]);
03465
03466 src1+=4;
03467 src2+=4;
03468 } while (src1 < src1_end);
03469 # endif
03470 #endif
03471 return count;
03472 }
03473
03474
03475
03476
03477
03478
03479
03480
03481
03482
03483
03484
03485 inline
03486 unsigned bit_block_xor_any(const bm::word_t* BMRESTRICT src1,
03487 const bm::word_t* BMRESTRICT src1_end,
03488 const bm::word_t* BMRESTRICT src2)
03489 {
03490 unsigned count = 0;
03491 do
03492 {
03493 count = (src1[0] ^ src2[0]) |
03494 (src1[1] ^ src2[1]) |
03495 (src1[2] ^ src2[2]) |
03496 (src1[3] ^ src2[3]);
03497
03498 src1+=4; src2+=4;
03499 } while ((src1 < src1_end) && (count == 0));
03500 return count;
03501 }
03502
03503
03504
03505
03506
03507
03508
03509
03510
03511
03512
03513
03514
03515
03516 inline
03517 unsigned bit_block_sub_count(const bm::word_t* BMRESTRICT src1,
03518 const bm::word_t* BMRESTRICT src1_end,
03519 const bm::word_t* BMRESTRICT src2)
03520 {
03521 unsigned count;
03522 #ifdef BMVECTOPT
03523 count = VECT_BITCOUNT_SUB(src1, src1_end, src2);
03524 #else
03525 count = 0;
03526 # ifdef BM64OPT
03527 const bm::id64_t* b1 = (bm::id64_t*) src1;
03528 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
03529 const bm::id64_t* b2 = (bm::id64_t*) src2;
03530 do
03531 {
03532 count += bitcount64_4way(b1[0] & ~b2[0],
03533 b1[1] & ~b2[1],
03534 b1[2] & ~b2[2],
03535 b1[3] & ~b2[3]);
03536 b1 += 4;
03537 b2 += 4;
03538 } while (b1 < b1_end);
03539 # else
03540 do
03541 {
03542 BM_INCWORD_BITCOUNT(count, src1[0] & ~src2[0]);
03543 BM_INCWORD_BITCOUNT(count, src1[1] & ~src2[1]);
03544 BM_INCWORD_BITCOUNT(count, src1[2] & ~src2[2]);
03545 BM_INCWORD_BITCOUNT(count, src1[3] & ~src2[3]);
03546
03547 src1+=4;
03548 src2+=4;
03549 } while (src1 < src1_end);
03550 # endif
03551 #endif
03552 return count;
03553 }
03554
03555
03556
03557
03558
03559
03560
03561
03562
03563
03564
03565 inline
03566 unsigned bit_block_sub_any(const bm::word_t* BMRESTRICT src1,
03567 const bm::word_t* BMRESTRICT src1_end,
03568 const bm::word_t* BMRESTRICT src2)
03569 {
03570 unsigned count = 0;
03571 do
03572 {
03573 count = (src1[0] & ~src2[0]) |
03574 (src1[1] & ~src2[1]) |
03575 (src1[2] & ~src2[2]) |
03576 (src1[3] & ~src2[3]);
03577
03578 src1+=4; src2+=4;
03579 } while ((src1 < src1_end) && (count == 0));
03580 return count;
03581 }
03582
03583
03584
03585
03586
03587
03588
03589
03590
03591
03592
03593
03594
03595 inline
03596 unsigned bit_block_or_count(const bm::word_t* src1,
03597 const bm::word_t* src1_end,
03598 const bm::word_t* src2)
03599 {
03600 unsigned count;
03601 #ifdef BMVECTOPT
03602 count = VECT_BITCOUNT_OR(src1, src1_end, src2);
03603 #else
03604 count = 0;
03605 # ifdef BM64OPT
03606 const bm::id64_t* b1 = (bm::id64_t*) src1;
03607 const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
03608 const bm::id64_t* b2 = (bm::id64_t*) src2;
03609 do
03610 {
03611 count += bitcount64_4way(b1[0] | b2[0],
03612 b1[1] | b2[1],
03613 b1[2] | b2[2],
03614 b1[3] | b2[3]);
03615 b1 += 4;
03616 b2 += 4;
03617 } while (b1 < b1_end);
03618 # else
03619 do
03620 {
03621 BM_INCWORD_BITCOUNT(count, src1[0] | src2[0]);
03622 BM_INCWORD_BITCOUNT(count, src1[1] | src2[1]);
03623 BM_INCWORD_BITCOUNT(count, src1[2] | src2[2]);
03624 BM_INCWORD_BITCOUNT(count, src1[3] | src2[3]);
03625
03626 src1+=4;
03627 src2+=4;
03628 } while (src1 < src1_end);
03629 # endif
03630 #endif
03631 return count;
03632 }
03633
03634
03635
03636
03637
03638
03639
03640
03641
03642
03643
03644 inline
03645 unsigned bit_block_or_any(const bm::word_t* BMRESTRICT src1,
03646 const bm::word_t* BMRESTRICT src1_end,
03647 const bm::word_t* BMRESTRICT src2)
03648 {
03649 unsigned count = 0;
03650 do
03651 {
03652 count = (src1[0] | src2[0]) |
03653 (src1[1] | src2[1]) |
03654 (src1[2] | src2[2]) |
03655 (src1[3] | src2[3]);
03656
03657 src1+=4; src2+=4;
03658 } while ((src1 < src1_end) && (count == 0));
03659 return count;
03660 }
03661
03662
03663
03664
03665
03666
03667
03668
03669
03670
03671
03672
03673
03674
03675
03676
03677 inline bm::word_t* bit_operation_and(bm::word_t* BMRESTRICT dst,
03678 const bm::word_t* BMRESTRICT src)
03679 {
03680 BM_ASSERT(dst || src);
03681
03682 bm::word_t* ret = dst;
03683
03684 if (IS_VALID_ADDR(dst))
03685 {
03686
03687 if (!IS_VALID_ADDR(src))
03688 {
03689 if (IS_EMPTY_BLOCK(src))
03690 {
03691
03692
03693 return 0;
03694 }
03695 }
03696 else
03697 {
03698
03699 bit_block_and(dst, src);
03700 }
03701 }
03702 else
03703 {
03704 if(!IS_VALID_ADDR(src))
03705 {
03706 if(IS_EMPTY_BLOCK(src))
03707 {
03708
03709
03710 return 0;
03711 }
03712
03713
03714 }
03715 else
03716 {
03717 if (IS_FULL_BLOCK(dst))
03718 {
03719 return const_cast<bm::word_t*>(src);
03720 }
03721
03722
03723 }
03724 }
03725
03726 return ret;
03727 }
03728
03729
03730
03731
03732
03733
03734
03735
03736
03737
03738
03739
03740
03741 inline
03742 bm::id_t bit_operation_and_count(const bm::word_t* BMRESTRICT src1,
03743 const bm::word_t* BMRESTRICT src1_end,
03744 const bm::word_t* BMRESTRICT src2)
03745 {
03746 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
03747 {
03748 return 0;
03749 }
03750 return bit_block_and_count(src1, src1_end, src2);
03751 }
03752
03753
03754
03755
03756
03757
03758
03759
03760
03761
03762
03763
03764 inline
03765 bm::id_t bit_operation_and_any(const bm::word_t* BMRESTRICT src1,
03766 const bm::word_t* BMRESTRICT src1_end,
03767 const bm::word_t* BMRESTRICT src2)
03768 {
03769 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
03770 {
03771 return 0;
03772 }
03773 return bit_block_and_any(src1, src1_end, src2);
03774 }
03775
03776
03777
03778
03779
03780
03781
03782
03783
03784
03785
03786
03787
03788
03789 inline
03790 bm::id_t bit_operation_sub_count(const bm::word_t* BMRESTRICT src1,
03791 const bm::word_t* BMRESTRICT src1_end,
03792 const bm::word_t* BMRESTRICT src2)
03793 {
03794 if (IS_EMPTY_BLOCK(src1))
03795 {
03796 return 0;
03797 }
03798
03799 if (IS_EMPTY_BLOCK(src2))
03800 {
03801 return bit_block_calc_count(src1, src1_end);
03802 }
03803 return bit_block_sub_count(src1, src1_end, src2);
03804 }
03805
03806
03807
03808
03809
03810
03811
03812
03813
03814
03815
03816
03817
03818
03819 inline
03820 bm::id_t bit_operation_sub_count_inv(const bm::word_t* BMRESTRICT src1,
03821 const bm::word_t* BMRESTRICT src1_end,
03822 const bm::word_t* BMRESTRICT src2)
03823 {
03824 unsigned arr_size = unsigned(src1_end - src1);
03825 return bit_operation_sub_count(src2, src2+arr_size, src1);
03826 }
03827
03828
03829
03830
03831
03832
03833
03834
03835
03836
03837
03838
03839
03840 inline
03841 bm::id_t bit_operation_sub_any(const bm::word_t* BMRESTRICT src1,
03842 const bm::word_t* BMRESTRICT src1_end,
03843 const bm::word_t* BMRESTRICT src2)
03844 {
03845 if (IS_EMPTY_BLOCK(src1))
03846 {
03847 return 0;
03848 }
03849
03850 if (IS_EMPTY_BLOCK(src2))
03851 {
03852 return !bit_is_all_zero((bm::wordop_t*)src1, (bm::wordop_t*)src1_end);
03853 }
03854 return bit_block_sub_any(src1, src1_end, src2);
03855 }
03856
03857
03858
03859
03860
03861
03862
03863
03864
03865
03866
03867
03868
03869
03870 inline
03871 bm::id_t bit_operation_or_count(const bm::word_t* BMRESTRICT src1,
03872 const bm::word_t* BMRESTRICT src1_end,
03873 const bm::word_t* BMRESTRICT src2)
03874 {
03875 if (IS_EMPTY_BLOCK(src1))
03876 {
03877 if (!IS_EMPTY_BLOCK(src2))
03878 return bit_block_calc_count(src2, src2 + (src1_end - src1));
03879 else
03880 return 0;
03881 }
03882 else
03883 {
03884 if (IS_EMPTY_BLOCK(src2))
03885 return bit_block_calc_count(src1, src1_end);
03886 }
03887
03888 return bit_block_or_count(src1, src1_end, src2);
03889 }
03890
03891
03892
03893
03894
03895
03896
03897
03898
03899
03900
03901
03902 inline
03903 bm::id_t bit_operation_or_any(const bm::word_t* BMRESTRICT src1,
03904 const bm::word_t* BMRESTRICT src1_end,
03905 const bm::word_t* BMRESTRICT src2)
03906 {
03907 if (IS_EMPTY_BLOCK(src1))
03908 {
03909 if (!IS_EMPTY_BLOCK(src2))
03910 return !bit_is_all_zero((bm::wordop_t*)src2,
03911 (bm::wordop_t*)(src2 + (src1_end - src1)));
03912 else
03913 return 0;
03914 }
03915 else
03916 {
03917 if (IS_EMPTY_BLOCK(src2))
03918 return !bit_is_all_zero((bm::wordop_t*)src1, (bm::wordop_t*)src1_end);
03919 }
03920
03921 return bit_block_or_any(src1, src1_end, src2);
03922 }
03923
03924
03925
03926
03927
03928
03929
03930
03931
03932
03933
03934
03935 inline void bit_block_or(bm::word_t* BMRESTRICT dst,
03936 const bm::word_t* BMRESTRICT src)
03937 {
03938 #ifdef BMVECTOPT
03939 VECT_OR_ARR(dst, src, src + bm::set_block_size);
03940 #else
03941 const bm::wordop_t* BMRESTRICT wrd_ptr = (wordop_t*)src;
03942 const bm::wordop_t* BMRESTRICT wrd_end = (wordop_t*)(src + set_block_size);
03943 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
03944
03945 do
03946 {
03947 dst_ptr[0] |= wrd_ptr[0];
03948 dst_ptr[1] |= wrd_ptr[1];
03949 dst_ptr[2] |= wrd_ptr[2];
03950 dst_ptr[3] |= wrd_ptr[3];
03951
03952 dst_ptr+=4;
03953 wrd_ptr+=4;
03954
03955 } while (wrd_ptr < wrd_end);
03956 #endif
03957 }
03958
03959
03960
03961
03962
03963
03964
03965
03966
03967
03968
03969
03970
03971
03972 inline
03973 bm::word_t* bit_operation_or(bm::word_t* BMRESTRICT dst,
03974 const bm::word_t* BMRESTRICT src)
03975 {
03976 BM_ASSERT(dst || src);
03977
03978 bm::word_t* ret = dst;
03979
03980 if (IS_VALID_ADDR(dst))
03981 {
03982 if (!IS_VALID_ADDR(src))
03983 {
03984 if (IS_FULL_BLOCK(src))
03985 {
03986
03987
03988 ::memset(dst, 0xFF, bm::set_block_size * sizeof(bm::word_t));
03989 }
03990 }
03991 else
03992 {
03993
03994 bit_block_or(dst, src);
03995 }
03996 }
03997 else
03998 {
03999 if (!IS_VALID_ADDR(src))
04000 {
04001 if (IS_FULL_BLOCK(src))
04002 {
04003
04004
04005 return const_cast<bm::word_t*>(FULL_BLOCK_ADDR);
04006 }
04007 }
04008 else
04009 {
04010 if (dst == 0)
04011 {
04012
04013
04014 return const_cast<bm::word_t*>(src);
04015 }
04016 }
04017 }
04018 return ret;
04019 }
04020
04021
04022
04023
04024
04025
04026
04027
04028
04029
04030 inline
04031 void bit_block_sub(bm::word_t* BMRESTRICT dst,
04032 const bm::word_t* BMRESTRICT src)
04033 {
04034 #ifdef BMVECTOPT
04035 VECT_SUB_ARR(dst, src, src + bm::set_block_size);
04036 #else
04037 const bm::wordop_t* BMRESTRICT wrd_ptr = (wordop_t*) src;
04038 const bm::wordop_t* BMRESTRICT wrd_end =
04039 (wordop_t*) (src + bm::set_block_size);
04040 bm::wordop_t* dst_ptr = (wordop_t*)dst;
04041
04042
04043 do
04044 {
04045 dst_ptr[0] &= ~wrd_ptr[0];
04046 dst_ptr[1] &= ~wrd_ptr[1];
04047 dst_ptr[2] &= ~wrd_ptr[2];
04048 dst_ptr[3] &= ~wrd_ptr[3];
04049
04050 dst_ptr+=4;
04051 wrd_ptr+=4;
04052 } while (wrd_ptr < wrd_end);
04053 #endif
04054
04055 }
04056
04057
04058
04059
04060
04061
04062
04063
04064
04065
04066
04067
04068
04069
04070 inline
04071 bm::word_t* bit_operation_sub(bm::word_t* BMRESTRICT dst,
04072 const bm::word_t* BMRESTRICT src)
04073 {
04074 BM_ASSERT(dst || src);
04075
04076 bm::word_t* ret = dst;
04077 if (IS_VALID_ADDR(dst))
04078 {
04079 if (!IS_VALID_ADDR(src))
04080 {
04081 if (IS_FULL_BLOCK(src))
04082 {
04083
04084
04085 return 0;
04086 }
04087 }
04088 else
04089 {
04090 bit_block_sub(dst, src);
04091 }
04092 }
04093 else
04094 {
04095 if (!IS_VALID_ADDR(src))
04096 {
04097 if (IS_FULL_BLOCK(src))
04098 {
04099
04100 return 0;
04101 }
04102 }
04103 else
04104 {
04105 if (IS_FULL_BLOCK(dst))
04106 {
04107
04108
04109 return const_cast<bm::word_t*>(src);
04110 }
04111 }
04112 }
04113 return ret;
04114 }
04115
04116
04117
04118
04119
04120
04121
04122
04123
04124
04125
04126 inline
04127 void bit_block_xor(bm::word_t* BMRESTRICT dst,
04128 const bm::word_t* BMRESTRICT src)
04129 {
04130 #ifdef BMVECTOPT
04131 VECT_XOR_ARR(dst, src, src + bm::set_block_size);
04132 #else
04133 const bm::wordop_t* BMRESTRICT wrd_ptr = (wordop_t*) src;
04134 const bm::wordop_t* BMRESTRICT wrd_end =
04135 (wordop_t*) (src + bm::set_block_size);
04136 bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
04137
04138
04139 do
04140 {
04141 dst_ptr[0] ^= wrd_ptr[0];
04142 dst_ptr[1] ^= wrd_ptr[1];
04143 dst_ptr[2] ^= wrd_ptr[2];
04144 dst_ptr[3] ^= wrd_ptr[3];
04145
04146 dst_ptr+=4;
04147 wrd_ptr+=4;
04148 } while (wrd_ptr < wrd_end);
04149 #endif
04150
04151 }
04152
04153
04154
04155
04156
04157
04158
04159
04160
04161
04162
04163
04164
04165
04166 inline
04167 bm::word_t* bit_operation_xor(bm::word_t* BMRESTRICT dst,
04168 const bm::word_t* BMRESTRICT src)
04169 {
04170 BM_ASSERT(dst || src);
04171 if (src == dst) return 0;
04172
04173 bm::word_t* ret = dst;
04174
04175 if (IS_VALID_ADDR(dst))
04176 {
04177 if (!src) return dst;
04178
04179 bit_block_xor(dst, src);
04180 }
04181 else
04182 {
04183 if (!src) return dst;
04184
04185
04186
04187
04188
04189 return const_cast<bm::word_t*>(src);
04190 }
04191 return ret;
04192 }
04193
04194
04195
04196
04197
04198
04199
04200
04201
04202
04203
04204 inline
04205 bm::id_t bit_operation_xor_count(const bm::word_t* BMRESTRICT src1,
04206 const bm::word_t* BMRESTRICT src1_end,
04207 const bm::word_t* BMRESTRICT src2)
04208 {
04209 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
04210 {
04211 if (IS_EMPTY_BLOCK(src1) && IS_EMPTY_BLOCK(src2))
04212 return 0;
04213 const bm::word_t* block = IS_EMPTY_BLOCK(src1) ? src2 : src1;
04214 return bit_block_calc_count(block, block + (src1_end - src1));
04215 }
04216 return bit_block_xor_count(src1, src1_end, src2);
04217 }
04218
04219
04220
04221
04222
04223
04224
04225
04226
04227
04228
04229 inline
04230 bm::id_t bit_operation_xor_any(const bm::word_t* BMRESTRICT src1,
04231 const bm::word_t* BMRESTRICT src1_end,
04232 const bm::word_t* BMRESTRICT src2)
04233 {
04234 if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
04235 {
04236 if (IS_EMPTY_BLOCK(src1) && IS_EMPTY_BLOCK(src2))
04237 return 0;
04238 const bm::word_t* block = IS_EMPTY_BLOCK(src1) ? src2 : src1;
04239 return !bit_is_all_zero((bm::wordop_t*)block,
04240 (bm::wordop_t*)(block + (src1_end - src1)));
04241 }
04242 return bit_block_xor_any(src1, src1_end, src2);
04243 }
04244
04245
04246
04247
04248
04249
04250
04251
04252
04253
04254
04255
04256
04257
04258 template<class T>
04259 unsigned bit_count_nonzero_size(const T* blk,
04260 unsigned data_size)
04261 {
04262 BM_ASSERT(blk && data_size);
04263 unsigned count = 0;
04264 const T* blk_end = blk + data_size - 2;
04265
04266 do
04267 {
04268 if (*blk == 0)
04269 {
04270
04271 const T* blk_j = blk + 1;
04272 for (; blk_j < blk_end; ++blk_j)
04273 {
04274 if (*blk_j != 0)
04275 break;
04276 }
04277 blk = blk_j-1;
04278 count += sizeof(gap_word_t);
04279 }
04280 else
04281 {
04282
04283 const T* blk_j = blk + 1;
04284 for ( ; blk_j < blk_end; ++blk_j)
04285 {
04286 if (*blk_j == 0)
04287 {
04288
04289 if (blk_j[1] | blk_j[2])
04290 {
04291
04292 ++blk_j;
04293 continue;
04294 }
04295 break;
04296 }
04297 }
04298 count += sizeof(gap_word_t);
04299
04300 count += (blk_j - blk) * sizeof(T);
04301 blk = blk_j;
04302 }
04303 ++blk;
04304 }
04305 while(blk < blk_end);
04306
04307 return count + (2 * sizeof(T));
04308 }
04309
04310
04311
04312
04313
04314
04315
04316
04317
04318
04319 inline
04320 int bit_find_in_block(const bm::word_t* data,
04321 unsigned nbit,
04322 bm::id_t* prev)
04323 {
04324 register bm::id_t p = *prev;
04325 int found = 0;
04326
04327 for(;;)
04328 {
04329 unsigned nword = nbit >> bm::set_word_shift;
04330 if (nword >= bm::set_block_size) break;
04331
04332 register bm::word_t val = data[nword] >> (p & bm::set_word_mask);
04333
04334
04335
04336
04337 if (val)
04338 {
04339 while((val & 1) == 0)
04340 {
04341 val >>= 1;
04342 ++nbit;
04343 ++p;
04344 }
04345 ++found;
04346
04347 break;
04348 }
04349 else
04350 {
04351 p += (bm::set_word_mask + 1) - (nbit & bm::set_word_mask);
04352 nbit += (bm::set_word_mask + 1) - (nbit & bm::set_word_mask);
04353 }
04354 }
04355 *prev = p;
04356 return found;
04357 }
04358
04359
04360
04361
04362
04363
04364
04365
04366 template<typename T, typename F>
04367 void bit_for_each_4(T w, F& func)
04368 {
04369 for (unsigned sub_octet = 0; w != 0; w >>= 4, sub_octet += 4)
04370 {
04371 switch (w & 15)
04372 {
04373 case 0:
04374 break;
04375 case 1:
04376 func(sub_octet);
04377 break;
04378 case 2:
04379 func(sub_octet + 1);
04380 break;
04381 case 3:
04382 func(sub_octet, sub_octet + 1);
04383 break;
04384 case 4:
04385 func(sub_octet + 2);
04386 break;
04387 case 5:
04388 func(sub_octet, sub_octet + 2);
04389 break;
04390 case 6:
04391 func(sub_octet + 1, sub_octet + 2);
04392 break;
04393 case 7:
04394 func(sub_octet, sub_octet + 1, sub_octet + 2);
04395 break;
04396 case 8:
04397 func(sub_octet + 3);
04398 break;
04399 case 9:
04400 func(sub_octet, sub_octet + 3);
04401 break;
04402 case 10:
04403 func(sub_octet + 1, sub_octet + 3);
04404 break;
04405 case 11:
04406 func(sub_octet, sub_octet + 1, sub_octet + 3);
04407 break;
04408 case 12:
04409 func(sub_octet + 2, sub_octet + 3);
04410 break;
04411 case 13:
04412 func(sub_octet, sub_octet + 2, sub_octet + 3);
04413 break;
04414 case 14:
04415 func(sub_octet + 1, sub_octet + 2, sub_octet + 3);
04416 break;
04417 case 15:
04418 func(sub_octet, sub_octet + 1, sub_octet + 2, sub_octet + 3);
04419 break;
04420 default:
04421 BM_ASSERT(0);
04422 break;
04423 }
04424
04425 }
04426 }
04427
04428
04429
04430
04431
04432
04433
04434
04435
04436 template<typename T, typename F>
04437 void bit_for_each(T w, F& func)
04438 {
04439
04440 for (unsigned octet = 0; w != 0; w >>= 8, octet += 8)
04441 {
04442 if (w & 1) func(octet + 0);
04443 if (w & 2) func(octet + 1);
04444 if (w & 4) func(octet + 2);
04445 if (w & 8) func(octet + 3);
04446 if (w & 16) func(octet + 4);
04447 if (w & 32) func(octet + 5);
04448 if (w & 64) func(octet + 6);
04449 if (w & 128) func(octet + 7);
04450
04451 }
04452 }
04453
04454
04455
04456
04457 template<typename B> class copy_to_array_functor
04458 {
04459 public:
04460 copy_to_array_functor(B* bits): bp_(bits)
04461 {}
04462
04463 B* ptr() { return bp_; }
04464
04465 void operator()(unsigned bit_idx) { *bp_++ = (B)bit_idx; }
04466
04467 void operator()(unsigned bit_idx0,
04468 unsigned bit_idx1)
04469 {
04470 bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1;
04471 bp_+=2;
04472 }
04473
04474 void operator()(unsigned bit_idx0,
04475 unsigned bit_idx1,
04476 unsigned bit_idx2)
04477 {
04478 bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1; bp_[2] = (B)bit_idx2;
04479 bp_+=3;
04480 }
04481
04482 void operator()(unsigned bit_idx0,
04483 unsigned bit_idx1,
04484 unsigned bit_idx2,
04485 unsigned bit_idx3)
04486 {
04487 bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1;
04488 bp_[2] = (B)bit_idx2; bp_[3] = (B)bit_idx3;
04489 bp_+=4;
04490 }
04491
04492 private:
04493 copy_to_array_functor(const copy_to_array_functor&);
04494 copy_to_array_functor& operator=(const copy_to_array_functor&);
04495 private:
04496 B* bp_;
04497 };
04498
04499
04500
04501
04502 template<typename B> class copy_to_array_functor_inc
04503 {
04504 public:
04505 copy_to_array_functor_inc(B* bits, unsigned base_idx)
04506 : bp_(bits), base_idx_(base_idx)
04507 {}
04508
04509 B* ptr() { return bp_; }
04510
04511 void operator()(unsigned bit_idx)
04512 {
04513 *bp_++ = (B)(bit_idx + base_idx_);
04514 }
04515
04516
04517 void operator()(unsigned bit_idx0,
04518 unsigned bit_idx1)
04519 {
04520 bp_[0]=(B)(bit_idx0+base_idx_);bp_[1]=(B)(bit_idx1+base_idx_);
04521 bp_+=2;
04522 }
04523
04524 void operator()(unsigned bit_idx0,
04525 unsigned bit_idx1,
04526 unsigned bit_idx2)
04527 {
04528 bp_[0]=(B)(bit_idx0+base_idx_);bp_[1]=(B)(bit_idx1+base_idx_);
04529 bp_[2]=(B)(bit_idx2+base_idx_);
04530 bp_+=3;
04531 }
04532
04533 void operator()(unsigned bit_idx0,
04534 unsigned bit_idx1,
04535 unsigned bit_idx2,
04536 unsigned bit_idx3)
04537 {
04538 bp_[0]=(B)(bit_idx0+base_idx_);bp_[1]=(B)(bit_idx1+base_idx_);
04539 bp_[2]=(B)(bit_idx2+base_idx_);bp_[3]=(B)(bit_idx3+base_idx_);
04540 bp_+=4;
04541 }
04542
04543 private:
04544 copy_to_array_functor_inc(const copy_to_array_functor_inc&);
04545 copy_to_array_functor_inc& operator=(const copy_to_array_functor_inc&);
04546 private:
04547 B* bp_;
04548 unsigned base_idx_;
04549 };
04550
04551
04552
04553
04554
04555
04556
04557
04558
04559
04560 template<typename T,typename B> unsigned bit_list_4(T w, B* bits)
04561 {
04562 copy_to_array_functor<B> func(bits);
04563 bit_for_each_4(w, func);
04564 return (unsigned)(func.ptr() - bits);
04565 }
04566
04567
04568
04569
04570
04571
04572
04573
04574
04575 template<typename T,typename B> unsigned bit_list(T w, B* bits)
04576 {
04577 copy_to_array_functor<B> func(bits);
04578 bit_for_each(w, func);
04579 return (unsigned)(func.ptr() - bits);
04580 }
04581
04582
04583
04584
04585
04586
04587 inline
04588 bm::set_representation best_representation(unsigned bit_count,
04589 unsigned total_possible_bitcount,
04590 unsigned gap_count,
04591 unsigned block_size)
04592 {
04593 unsigned arr_size = sizeof(bm::gap_word_t) * bit_count + sizeof(bm::gap_word_t);
04594 unsigned gap_size = sizeof(bm::gap_word_t) * gap_count + sizeof(bm::gap_word_t);
04595 unsigned inv_arr_size = sizeof(bm::gap_word_t) * (total_possible_bitcount - bit_count) + sizeof(bm::gap_word_t);
04596
04597 if ((gap_size < block_size) && (gap_size < arr_size) && (gap_size < inv_arr_size))
04598 {
04599 return bm::set_gap;
04600 }
04601
04602 if (arr_size < inv_arr_size)
04603 {
04604 if ((arr_size < block_size) && (arr_size < gap_size))
04605 {
04606 return bm::set_array1;
04607 }
04608 }
04609 else
04610 {
04611 if ((inv_arr_size < block_size) && (inv_arr_size < gap_size))
04612 {
04613 return bm::set_array0;
04614 }
04615 }
04616 return bm::set_bitset;
04617 }
04618
04619
04620
04621
04622
04623 template<typename T> T bit_convert_to_arr(T* BMRESTRICT dest,
04624 const unsigned* BMRESTRICT src,
04625 bm::id_t bits,
04626 unsigned dest_len,
04627 unsigned mask = 0)
04628 {
04629 T* BMRESTRICT pcurr = dest;
04630 for(unsigned bit_idx=0; bit_idx < bits; ++src,bit_idx += sizeof(*src) * 8)
04631 {
04632 unsigned val = *src ^ mask;
04633 if (val == 0)
04634 {
04635 continue;
04636 }
04637 if (pcurr + sizeof(val)*8 >= dest + dest_len)
04638 {
04639 return 0;
04640 }
04641
04642 copy_to_array_functor_inc<T> func(pcurr, bit_idx);
04643 bit_for_each_4(val, func);
04644 unsigned word_bit_cnt = func.ptr() - pcurr;
04645 pcurr += word_bit_cnt;
04646
04647 }
04648 return (T)(pcurr - dest);
04649 }
04650
04651
04652
04653
04654
04655
04656
04657
04658
04659
04660
04661
04662
04663
04664
04665
04666
04667
04668
04669
04670
04671
04672
04673
04674
04675
04676
04677
04678
04679
04680
04681
04682
04683
04684
04685
04686
04687
04688
04689
04690
04691
04692
04693
04694
04695
04696
04697
04698
04699
04700
04701
04702
04703
04704
04705
04706
04707
04708
04709
04710
04711
04712
04713
04714
04715
04716
04717
04718
04719
04720
04721 template<typename T>
04722 unsigned gap_overhead(const T* length,
04723 const T* length_end,
04724 const T* glevel_len)
04725 {
04726 BM_ASSERT(length && length_end && glevel_len);
04727
04728 unsigned overhead = 0;
04729 for (;length < length_end; ++length)
04730 {
04731 unsigned len = *length;
04732 int level = gap_calc_level(len, glevel_len);
04733 BM_ASSERT(level >= 0 && level < (int)bm::gap_levels);
04734 unsigned capacity = glevel_len[level];
04735 BM_ASSERT(capacity >= len);
04736 overhead += capacity - len;
04737 }
04738 return overhead;
04739 }
04740
04741
04742
04743
04744
04745
04746
04747
04748 template<typename T>
04749 bool improve_gap_levels(const T* length,
04750 const T* length_end,
04751 T* glevel_len)
04752 {
04753 BM_ASSERT(length && length_end && glevel_len);
04754
04755 size_t lsize = length_end - length;
04756
04757 BM_ASSERT(lsize);
04758
04759 gap_word_t max_len = 0;
04760 unsigned i;
04761 for (i = 0; i < lsize; ++i)
04762 {
04763 if (length[i] > max_len)
04764 max_len = length[i];
04765 }
04766 if (max_len < 5 || lsize <= bm::gap_levels)
04767 {
04768 glevel_len[0] = max_len + 4;
04769 for (i = 1; i < bm::gap_levels; ++i)
04770 {
04771 glevel_len[i] = bm::gap_max_buff_len;
04772 }
04773 return true;
04774 }
04775
04776 glevel_len[bm::gap_levels-1] = max_len + 5;
04777
04778 unsigned min_overhead = gap_overhead(length, length_end, glevel_len);
04779 bool is_improved = false;
04780 gap_word_t prev_value = glevel_len[bm::gap_levels-1];
04781
04782
04783
04784 for (i = bm::gap_levels-2; ; --i)
04785 {
04786 unsigned opt_len = 0;
04787 unsigned j;
04788 bool imp_flag = false;
04789 gap_word_t gap_saved_value = glevel_len[i];
04790 for (j = 0; j < lsize; ++j)
04791 {
04792 glevel_len[i] = length[j]+4;
04793 unsigned ov = gap_overhead(length, length_end, glevel_len);
04794 if (ov <= min_overhead)
04795 {
04796 min_overhead = ov;
04797 opt_len = length[j]+4;
04798 imp_flag = true;
04799 }
04800 }
04801 if (imp_flag) {
04802 glevel_len[i] = opt_len;
04803 is_improved = true;
04804 }
04805 else
04806 {
04807 glevel_len[i] = gap_saved_value;
04808 }
04809 if (i == 0)
04810 break;
04811 prev_value = glevel_len[i];
04812 }
04813
04814
04815
04816
04817 T val = *glevel_len;
04818 T* gp = glevel_len;
04819 T* res = glevel_len;
04820 for (i = 0; i < bm::gap_levels; ++i)
04821 {
04822 if (val != *gp)
04823 {
04824 val = *gp;
04825 *++res = val;
04826 }
04827 ++gp;
04828 }
04829
04830
04831 while (++res < (glevel_len + bm::gap_levels))
04832 {
04833 *res = bm::gap_max_buff_len;
04834 }
04835
04836 return is_improved;
04837
04838 }
04839
04840
04841
04842
04843
04844
04845
04846
04847 class bitblock_get_adapter
04848 {
04849 public:
04850 bitblock_get_adapter(const bm::word_t* bit_block) : b_(bit_block) {}
04851
04852 BMFORCEINLINE
04853 bm::word_t get_32() { return *b_++; }
04854 private:
04855 const bm::word_t* b_;
04856 };
04857
04858
04859
04860
04861
04862
04863 class bitblock_store_adapter
04864 {
04865 public:
04866 bitblock_store_adapter(bm::word_t* bit_block) : b_(bit_block) {}
04867 BMFORCEINLINE
04868 void push_back(bm::word_t w) { *b_++ = w; }
04869 private:
04870 bm::word_t* b_;
04871 };
04872
04873
04874
04875
04876
04877 class bitblock_sum_adapter
04878 {
04879 public:
04880 bitblock_sum_adapter() : sum_(0) {}
04881 BMFORCEINLINE
04882 void push_back(bm::word_t w) { this->sum_+= w; }
04883
04884 bm::word_t sum() const { return this->sum_; }
04885 private:
04886 bm::word_t sum_;
04887 };
04888
04889
04890
04891
04892
04893
04894 template<class DEC> class decoder_range_adapter
04895 {
04896 public:
04897 decoder_range_adapter(DEC& dec, unsigned from_idx, unsigned to_idx)
04898 : decoder_(dec),
04899 from_(from_idx),
04900 to_(to_idx),
04901 cnt_(0)
04902 {}
04903
04904 bm::word_t get_32()
04905 {
04906 if (cnt_ < from_ || cnt_ > to_)
04907 {
04908 ++cnt_; return 0;
04909 }
04910 ++cnt_;
04911 return decoder_.get_32();
04912 }
04913
04914 private:
04915 DEC& decoder_;
04916 unsigned from_;
04917 unsigned to_;
04918 unsigned cnt_;
04919 };
04920
04921
04922
04923
04924
04925
04926 template<class It1, class It2, class BinaryOp, class Encoder>
04927 void bit_recomb(It1& it1, It2& it2,
04928 BinaryOp& op,
04929 Encoder& enc,
04930 unsigned block_size = bm::set_block_size)
04931 {
04932 for (unsigned i = 0; i < block_size; ++i)
04933 {
04934 bm::word_t w1 = it1.get_32();
04935 bm::word_t w2 = it2.get_32();
04936 bm::word_t w = op(w1, w2);
04937 enc.push_back( w );
04938 }
04939 }
04940
04941
04942 template<typename W> struct bit_AND
04943 {
04944 W operator()(W w1, W w2) { return w1 & w2; }
04945 };
04946
04947
04948 template<typename W> struct bit_OR
04949 {
04950 W operator()(W w1, W w2) { return w1 | w2; }
04951 };
04952
04953
04954 template<typename W> struct bit_SUB
04955 {
04956 W operator()(W w1, W w2) { return w1 & ~w2; }
04957 };
04958
04959
04960 template<typename W> struct bit_XOR
04961 {
04962 W operator()(W w1, W w2) { return w1 ^ w2; }
04963 };
04964
04965
04966 template<typename W> struct bit_ASSIGN
04967 {
04968 W operator()(W w1, W w2) { return w2; }
04969 };
04970
04971
04972 template<typename W> struct bit_COUNT
04973 {
04974 W operator()(W w1, W w2)
04975 {
04976 w1 = 0;
04977 BM_INCWORD_BITCOUNT(w1, w2);
04978 return w1;
04979 }
04980 };
04981
04982
04983 template<typename W> struct bit_COUNT_AND
04984 {
04985 W operator()(W w1, W w2)
04986 {
04987 W r = 0;
04988 BM_INCWORD_BITCOUNT(r, w1 & w2);
04989 return r;
04990 }
04991 };
04992
04993
04994 template<typename W> struct bit_COUNT_XOR
04995 {
04996 W operator()(W w1, W w2)
04997 {
04998 W r = 0;
04999 BM_INCWORD_BITCOUNT(r, w1 ^ w2);
05000 return r;
05001 }
05002 };
05003
05004
05005 template<typename W> struct bit_COUNT_OR
05006 {
05007 W operator()(W w1, W w2)
05008 {
05009 W r = 0;
05010 BM_INCWORD_BITCOUNT(r, w1 | w2);
05011 return r;
05012 }
05013 };
05014
05015
05016
05017 template<typename W> struct bit_COUNT_SUB_AB
05018 {
05019 W operator()(W w1, W w2)
05020 {
05021 W r = 0;
05022 BM_INCWORD_BITCOUNT(r, w1 & (~w2));
05023 return r;
05024 }
05025 };
05026
05027
05028 template<typename W> struct bit_COUNT_SUB_BA
05029 {
05030 W operator()(W w1, W w2)
05031 {
05032 W r = 0;
05033 BM_INCWORD_BITCOUNT(r, w2 & (~w1));
05034 return r;
05035 }
05036 };
05037
05038
05039 template<typename W> struct bit_COUNT_A
05040 {
05041 W operator()(W w1, W w2)
05042 {
05043 W r = 0;
05044 BM_INCWORD_BITCOUNT(r, w1);
05045 return r;
05046 }
05047 };
05048
05049
05050 template<typename W> struct bit_COUNT_B
05051 {
05052 W operator()(W w1, W w2)
05053 {
05054 W r = 0;
05055 BM_INCWORD_BITCOUNT(r, w2);
05056 return r;
05057 }
05058 };
05059
05060 typedef
05061 void (*gap_operation_to_bitset_func_type)(unsigned*,
05062 const gap_word_t*);
05063
05064 typedef
05065 gap_word_t* (*gap_operation_func_type)(const gap_word_t* BMRESTRICT,
05066 const gap_word_t* BMRESTRICT,
05067 gap_word_t* BMRESTRICT,
05068 unsigned& );
05069
05070 typedef
05071 bm::id_t (*bit_operation_count_func_type)(const bm::word_t* BMRESTRICT,
05072 const bm::word_t* BMRESTRICT,
05073 const bm::word_t* BMRESTRICT);
05074
05075
05076 template<bool T>
05077 struct operation_functions
05078 {
05079 static
05080 gap_operation_to_bitset_func_type gap2bit_table_[bm::set_END];
05081 static
05082 gap_operation_func_type gapop_table_[bm::set_END];
05083 static
05084 bit_operation_count_func_type bit_op_count_table_[bm::set_END];
05085
05086 static
05087 gap_operation_to_bitset_func_type gap_op_to_bit(unsigned i)
05088 {
05089 return gap2bit_table_[i];
05090 }
05091
05092 static
05093 gap_operation_func_type gap_operation(unsigned i)
05094 {
05095 return gapop_table_[i];
05096 }
05097
05098 static
05099 bit_operation_count_func_type bit_operation_count(unsigned i)
05100 {
05101 return bit_op_count_table_[i];
05102 }
05103 };
05104
05105 template<bool T>
05106 gap_operation_to_bitset_func_type
05107 operation_functions<T>::gap2bit_table_[bm::set_END] = {
05108 &gap_and_to_bitset<bm::gap_word_t>,
05109 &gap_add_to_bitset<bm::gap_word_t>,
05110 &gap_sub_to_bitset<bm::gap_word_t>,
05111 &gap_xor_to_bitset<bm::gap_word_t>,
05112 0
05113 };
05114
05115 template<bool T>
05116 gap_operation_func_type
05117 operation_functions<T>::gapop_table_[bm::set_END] = {
05118 &gap_operation_and,
05119 &gap_operation_or,
05120 &gap_operation_sub,
05121 &gap_operation_xor,
05122 0
05123 };
05124
05125
05126 template<bool T>
05127 bit_operation_count_func_type
05128 operation_functions<T>::bit_op_count_table_[bm::set_END] = {
05129 0,
05130 0,
05131 0,
05132 0,
05133 0,
05134 0,
05135 &bit_operation_and_count,
05136 &bit_operation_xor_count,
05137 &bit_operation_or_count,
05138 &bit_operation_sub_count,
05139 &bit_operation_sub_count_inv,
05140 0,
05141 0,
05142 };
05143
05144 }
05145
05146 #endif