feat: Conversion to conventional bitset (#448)

* Conversion to conventional bitset implemented.

* Minor cleaning.

* Minor fixes

* Adding bitset to amalgamation.

* Various fixes

* Adding C++ guards.

* Various fixes

* Make free functions safe for NULL ptr

* Various fixes

* Correcting typo in previous commit (duplicated line)

* Corrected new typo

* We renamed "hamming" to "roaring_hamming"

* We are adopting roaring_trailing_zeroes

* Let us modernize our Visual Studio 16 2019 builds.

* Switching to windows-2019

* Various fixes... to the AVX-512 kernel, which is unexpected.

Author: lemire

.github/workflows/vs16-arm-ci.yml

@@ -8,17 +8,21 @@ permissions:
 jobs:
   ci:
     name: windows-vs16
-    runs-on: windows-latest
+    runs-on: windows-2019
     strategy:
       fail-fast: false
       matrix:
         include:
-          - {arch: ARM}
-          - {arch: ARM64}
+          - {gen: Visual Studio 16 2019, arch: ARM}
+          - {gen: Visual Studio 16 2019, arch: ARM64}
     steps:
       - name: checkout
         uses: actions/checkout@v2
-      - name: Use cmake
+      - name: Configure
         run: |
-          cmake -A ${{ matrix.arch }} -DCMAKE_CROSSCOMPILING=1 -B build  &&
-          cmake --build build --verbose
+          mkdir build
+          cd build && cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  ..
+      - name: Build
+        run: cmake --build build --config Release
+      - name: Build Debug
+        run: cmake --build build --config Debug

.github/workflows/vs16-ci.yml

@@ -8,21 +8,29 @@ permissions:
 jobs:
   ci:
     name: windows-vs16
-    runs-on: windows-latest
+    runs-on: windows-2019
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {gen: Visual Studio 16 2019, arch: Win32}
+          - {gen: Visual Studio 16 2019, arch: x64}
     steps:
-    - uses: actions/checkout@v2
-    - name: 'Run CMake with VS16'
-      uses: lukka/run-cmake@v2
-      with:
-        cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
-        cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
-        buildDirectory: "${{ github.workspace }}/../../_temp/windows"
-        cmakeBuildType: Release
-        buildWithCMake: true
-        cmakeGenerator: VS16Win64
-        cmakeAppendedArgs: -DROARING_BUILD_STATIC=ON
-        buildWithCMakeArgs: --config Release
-
-    - name: 'Run CTest'
-      run: ctest  --verbose
-      working-directory: "${{ github.workspace }}/../../_temp/windows"
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Configure
+        run: |
+          mkdir build
+          cd build && cmake -G "${{matrix.gen}}" -A ${{matrix.arch}}  ..
+      - name: Build
+        run: cmake --build build --config Release
+      - name: Run basic tests
+        run: |
+          cd build
+          ctest -C Release --output-on-failure
+      - name: Build Debug
+        run: cmake --build build --config Debug
+      - name: Run basic tests in Debug
+        run: |
+          cd build
+          ctest -C Debug --output-on-failure

README.md

@@ -349,6 +349,77 @@ int main() {
 }
 ```
 
+# Conventional bitsets (C)
+
+We support convention bitsets (uncompressed) as part of the library.
+
+Simple example:
+
+```C
+bitset_t * b = bitset_create();
+bitset_set(b,10);
+bitset_get(b,10);// returns true
+bitset_free(b); // frees memory
+```
+
+More advanced example:
+
+```C
+    bitset_t *b = bitset_create();
+    for (int k = 0; k < 1000; ++k) {
+        bitset_set(b, 3 * k);
+    }
+    // We have bitset_count(b) == 1000.
+    // We have bitset_get(b, 3) is true
+    // You can iterate through the values:
+    size_t k = 0;
+    for (size_t i = 0; bitset_next_set_bit(b, &i); i++) {
+        // You will have i == k
+        k += 3;
+    }
+    // We support a wide range of operations on two bitsets such as
+    // bitset_inplace_symmetric_difference(b1,b2);
+    // bitset_inplace_symmetric_difference(b1,b2);
+    // bitset_inplace_difference(b1,b2);// should make no difference
+    // bitset_inplace_union(b1,b2);
+    // bitset_inplace_intersection(b1,b2);
+    // bitsets_disjoint
+    // bitsets_intersect
+```
+
+In some instances, you may want to convert a Roaring bitmap into a conventional (uncompressed) bitset.
+Indeed, bitsets have advantages such as higher query performances in some cases. The following code
+illustrates how you may do so:
+
+```C
+    roaring_bitmap_t *r1 = roaring_bitmap_create();
+    for (uint32_t i = 100; i < 100000; i+= 1 + (i%5)) {
+     roaring_bitmap_add(r1, i);
+    }
+    for (uint32_t i = 100000; i < 500000; i+= 100) {
+     roaring_bitmap_add(r1, i);
+    }
+    roaring_bitmap_add_range(r1, 500000, 600000);
+    bitset_t * bitset = bitset_create();
+    bool success = roaring_bitmap_to_bitset(r1, bitset);
+    assert(success); // could fail due to memory allocation.
+    assert(bitset_count(bitset) == roaring_bitmap_get_cardinality(r1));
+    // You can then query the bitset:
+    for (uint32_t i = 100; i < 100000; i+= 1 + (i%5)) {
+        assert(bitset_get(bitset,i));
+    }
+    for (uint32_t i = 100000; i < 500000; i+= 100) {
+        assert(bitset_get(bitset,i));
+    }
+    // you must free the memory:
+    bitset_free(bitset);
+    roaring_bitmap_free(r1);
+```
+
+You should be aware that a convention bitset (`bitset_t *`) may use much more
+memory than a Roaring bitmap in some cases. You should run benchmarks to determine
+whether the conversion to a bitset has performance benefits in your case.
+
 # Example (C++)
 
 

amalgamation.sh

@@ -31,6 +31,8 @@ DEMOCPP="amalgamation_demo.cpp"
 ALL_PUBLIC_H="
 $SCRIPTPATH/include/roaring/roaring_version.h
 $SCRIPTPATH/include/roaring/roaring_types.h
+$SCRIPTPATH/include/roaring/portability.h
+$SCRIPTPATH/include/roaring/bitset/bitset.h
 $SCRIPTPATH/include/roaring/roaring.h
 $SCRIPTPATH/include/roaring/memory.h
 "
@@ -47,7 +49,6 @@ $SCRIPTPATH/cpp/roaring64map.hh
 # need to be in this order.
 #
 ALL_PRIVATE_H="
-$SCRIPTPATH/include/roaring/portability.h
 $SCRIPTPATH/include/roaring/isadetection.h
 $SCRIPTPATH/include/roaring/containers/perfparameters.h
 $SCRIPTPATH/include/roaring/containers/container_defs.h
@@ -165,13 +166,21 @@ echo "Creating ${DEMOC}..."
 
     cat <<< '
 #include <stdio.h>
+#include <stdlib.h>
 #include "roaring.c"
 int main() {
   roaring_bitmap_t *r1 = roaring_bitmap_create();
   for (uint32_t i = 100; i < 1000; i++) roaring_bitmap_add(r1, i);
   printf("cardinality = %d\n", (int) roaring_bitmap_get_cardinality(r1));
   roaring_bitmap_free(r1);
-  return 0;
+
+  bitset_t *b = bitset_create();
+  for (int k = 0; k < 1000; ++k) {
+        bitset_set(b, 3 * k);
+  }
+  printf("%zu \n", bitset_count(b));
+  bitset_free(b);
+  return EXIT_SUCCESS;
 }
 '
 } > "${DEMOC}"
@@ -241,10 +250,10 @@ CPPBIN=${DEMOCPP%%.*}
 echo "The interface is found in the file 'include/roaring/roaring.h'."
 newline
 echo "For C, try:"
-echo "cc -march=native -O3 -std=c11  -o ${CBIN} ${DEMOC}  && ./${CBIN} "
+echo "cc -O3 -std=c11  -o ${CBIN} ${DEMOC}  && ./${CBIN} "
 newline
 echo "For C++, try:"
-echo "c++ -march=native -O3 -std=c++11 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} "
+echo "c++ -O3 -std=c++11 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} "
 
 lowercase(){
     echo "$1" | tr 'A-Z' 'a-z'
@@ -256,8 +265,8 @@ newline
 echo "You can build a shared library with the following command:"
 
 if [ $OS == "darwin" ]; then
-  echo "cc -march=native -O3 -std=c11 -shared -o libroaring.dylib -fPIC roaring.c"
+  echo "cc  -O3 -std=c11 -shared -o libroaring.dylib -fPIC roaring.c"
 else
-  echo "cc -march=native -O3 -std=c11 -shared -o libroaring.so -fPIC roaring.c"
+  echo "cc -O3 -std=c11 -shared -o libroaring.so -fPIC roaring.c"
 fi