Fix sparseindices (#742)

* accept size_t for Vec

* Prepare a test case for "SparseIndices too large" core dump

* fix python docstring deps

* fix compilation errors

* limit sdf hashtable size

* another fix to hashtable

* log2(0) is undefined

* fix formatting

* Revert "fix compilation errors"

This reverts commit adfb86c.

* fix size_t stuff

* remove processOverlap setting in test

* fix hashtable

* vertex number limit

* address comments

Author: pca006132

CMakeLists.txt

@@ -86,7 +86,8 @@ if (MSVC)
   set(MANIFOLD_FLAGS ${MANIFOLD_FLAGS} /DNOMINMAX)
 else()
   if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    set(WARNING_FLAGS -Werror -Wall -Wno-sign-compare -Wno-unused -Wno-array-bounds -Wno-stringop-overflow)
+    set(WARNING_FLAGS -Werror -Wall -Wno-sign-compare -Wno-unused -Wno-array-bounds
+      -Wno-stringop-overflow -Wno-alloc-size-larger-than)
   else()
     set(WARNING_FLAGS -Werror -Wall -Wno-sign-compare -Wno-unused)
   endif()

bindings/python/CMakeLists.txt

@@ -26,8 +26,18 @@ target_compile_features(manifold3d PUBLIC cxx_std_17)
 set_target_properties(manifold3d PROPERTIES OUTPUT_NAME "manifold3d")
 
 message(Python_EXECUTABLE = ${Python_EXECUTABLE})
+# ideally we should generate a dependency file from python...
+set(DOCSTRING_DEPS
+  ${CMAKE_CURRENT_SOURCE_DIR}/../../src/manifold/src/manifold.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/../../src/manifold/src/constructors.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/../../src/manifold/src/sort.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/../../src/cross_section/src/cross_section.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/../../src/polygon/src/polygon.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utilities/include/public.h
+  )
 add_custom_command(
   OUTPUT autogen_docstrings.inl
+  DEPENDS ${DOCSTRING_DEPS}
   COMMAND ${Python_EXECUTABLE} ARGS ${CMAKE_CURRENT_SOURCE_DIR}/gen_docs.py
   WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
 )

src/manifold/include/manifold.h

@@ -46,9 +46,17 @@ class CsgLeafNode;
  */
 struct MeshGL {
   /// Number of property vertices
-  uint32_t NumVert() const { return vertProperties.size() / numProp; };
+  uint32_t NumVert() const {
+    if (vertProperties.size() / numProp >= std::numeric_limits<int>::max())
+      throw std::out_of_range("mesh too large");
+    return vertProperties.size() / numProp;
+  };
   /// Number of triangles
-  uint32_t NumTri() const { return triVerts.size() / 3; };
+  uint32_t NumTri() const {
+    if (vertProperties.size() / numProp >= std::numeric_limits<int>::max())
+      throw std::out_of_range("mesh too large");
+    return triVerts.size() / 3;
+  };
 
   /// Number of properties per vertex, always >= 3.
   uint32_t numProp = 3;

src/manifold/src/boolean3.cpp

@@ -71,9 +71,9 @@ struct CopyFaceEdges {
   SparseIndices &pXq1;
   VecView<const Halfedge> halfedgesQ;
 
-  void operator()(thrust::tuple<int, int> in) {
+  void operator()(thrust::tuple<size_t, size_t> in) {
     int idx = 3 * thrust::get<0>(in);
-    int i = thrust::get<1>(in);
+    size_t i = thrust::get<1>(in);
     int pX = p1q1.Get(i, inverted);
     int q2 = p1q1.Get(i, !inverted);
 
@@ -83,7 +83,7 @@ struct CopyFaceEdges {
       int a = pX;
       int b = edge.IsForward() ? q1 : edge.pairedHalfedge;
       if (inverted) std::swap(a, b);
-      pXq1.Set(idx + j, a, b);
+      pXq1.Set(idx + static_cast<size_t>(j), a, b);
     }
   }
 };
@@ -92,9 +92,9 @@ SparseIndices Filter11(const Manifold::Impl &inP, const Manifold::Impl &inQ,
                        const SparseIndices &p1q2, const SparseIndices &p2q1) {
   ZoneScoped;
   SparseIndices p1q1(3 * p1q2.size() + 3 * p2q1.size());
-  for_each_n(autoPolicy(p1q2.size()), zip(countAt(0), countAt(0)), p1q2.size(),
-             CopyFaceEdges<false>({p1q2, p1q1, inQ.halfedge_}));
-  for_each_n(autoPolicy(p2q1.size()), zip(countAt(p1q2.size()), countAt(0)),
+  for_each_n(autoPolicy(p1q2.size()), zip(countAt(0_z), countAt(0_z)),
+             p1q2.size(), CopyFaceEdges<false>({p1q2, p1q1, inQ.halfedge_}));
+  for_each_n(autoPolicy(p2q1.size()), zip(countAt(p1q2.size()), countAt(0_z)),
              p2q1.size(), CopyFaceEdges<true>({p2q1, p1q1, inP.halfedge_}));
   p1q1.Unique();
   return p1q1;
@@ -137,14 +137,14 @@ inline thrust::pair<int, glm::vec2> Shadow01(
 
 // https://github.com/scandum/binary_search/blob/master/README.md
 // much faster than standard binary search on large arrays
-int monobound_quaternary_search(VecView<const int64_t> array, int64_t key) {
+size_t monobound_quaternary_search(VecView<const int64_t> array, int64_t key) {
   if (array.size() == 0) {
     return -1;
   }
-  unsigned int bot = 0;
-  unsigned int top = array.size();
+  size_t bot = 0;
+  size_t top = array.size();
   while (top >= 65536) {
-    unsigned int mid = top / 4;
+    size_t mid = top / 4;
     top -= mid * 3;
     if (key < array[bot + mid * 2]) {
       if (key >= array[bot + mid]) {
@@ -159,7 +159,7 @@ int monobound_quaternary_search(VecView<const int64_t> array, int64_t key) {
   }
 
   while (top > 3) {
-    unsigned int mid = top / 2;
+    size_t mid = top / 2;
     if (key >= array[bot + mid]) {
       bot += mid;
     }
@@ -183,7 +183,7 @@ struct Kernel11 {
   VecView<const glm::vec3> normalP;
   const SparseIndices &p1q1;
 
-  void operator()(thrust::tuple<int, glm::vec4 &, int &> inout) {
+  void operator()(thrust::tuple<size_t, glm::vec4 &, int &> inout) {
     const int p1 = p1q1.Get(thrust::get<0>(inout), false);
     const int q1 = p1q1.Get(thrust::get<0>(inout), true);
     glm::vec4 &xyzz11 = thrust::get<1>(inout);
@@ -261,7 +261,7 @@ std::tuple<Vec<int>, Vec<glm::vec4>> Shadow11(SparseIndices &p1q1,
   Vec<glm::vec4> xyzz11(p1q1.size());
 
   for_each_n(autoPolicy(p1q1.size()),
-             zip(countAt(0), xyzz11.begin(), s11.begin()), p1q1.size(),
+             zip(countAt(0_z), xyzz11.begin(), s11.begin()), p1q1.size(),
              Kernel11({inP.vertPos_, inQ.vertPos_, inP.halfedge_, inQ.halfedge_,
                        expandP, inP.vertNormal_, p1q1}));
 
@@ -279,7 +279,7 @@ struct Kernel02 {
   const SparseIndices &p0q2;
   const bool forward;
 
-  void operator()(thrust::tuple<int, int &, float &> inout) {
+  void operator()(thrust::tuple<size_t, int &, float &> inout) {
     const int p0 = p0q2.Get(thrust::get<0>(inout), !forward);
     const int q2 = p0q2.Get(thrust::get<0>(inout), forward);
     int &s02 = thrust::get<1>(inout);
@@ -351,8 +351,8 @@ std::tuple<Vec<int>, Vec<float>> Shadow02(const Manifold::Impl &inP,
   Vec<float> z02(p0q2.size());
 
   auto vertNormalP = forward ? inP.vertNormal_ : inQ.vertNormal_;
-  for_each_n(autoPolicy(p0q2.size()), zip(countAt(0), s02.begin(), z02.begin()),
-             p0q2.size(),
+  for_each_n(autoPolicy(p0q2.size()),
+             zip(countAt(0_z), s02.begin(), z02.begin()), p0q2.size(),
              Kernel02({inP.vertPos_, inQ.halfedge_, inQ.vertPos_, expandP,
                        vertNormalP, p0q2, forward}));
 
@@ -374,7 +374,7 @@ struct Kernel12 {
   const bool forward;
   const SparseIndices &p1q2;
 
-  void operator()(thrust::tuple<int, int &, glm::vec3 &> inout) {
+  void operator()(thrust::tuple<size_t, int &, glm::vec3 &> inout) {
     int p1 = p1q2.Get(thrust::get<0>(inout), !forward);
     int q2 = p1q2.Get(thrust::get<0>(inout), forward);
     int &x12 = thrust::get<1>(inout);
@@ -394,7 +394,7 @@ struct Kernel12 {
     for (int vert : {edge.startVert, edge.endVert}) {
       const int64_t key = forward ? SparseIndices::EncodePQ(vert, q2)
                                   : SparseIndices::EncodePQ(q2, vert);
-      const int idx = monobound_quaternary_search(p0q2, key);
+      const size_t idx = monobound_quaternary_search(p0q2, key);
       if (idx != -1) {
         const int s = s02[idx];
         x12 += s * ((vert == edge.startVert) == forward ? 1 : -1);
@@ -415,7 +415,7 @@ struct Kernel12 {
       const int q1F = edge.IsForward() ? q1 : edge.pairedHalfedge;
       const int64_t key = forward ? SparseIndices::EncodePQ(p1, q1F)
                                   : SparseIndices::EncodePQ(q1F, p1);
-      const int idx = monobound_quaternary_search(p1q1, key);
+      const size_t idx = monobound_quaternary_search(p1q1, key);
       if (idx != -1) {  // s is implicitly zero for anything not found
         const int s = s11[idx];
         x12 -= s * (edge.IsForward() ? 1 : -1);
@@ -456,7 +456,7 @@ std::tuple<Vec<int>, Vec<glm::vec3>> Intersect12(
   Vec<glm::vec3> v12(p1q2.size());
 
   for_each_n(
-      autoPolicy(p1q2.size()), zip(countAt(0), x12.begin(), v12.begin()),
+      autoPolicy(p1q2.size()), zip(countAt(0_z), x12.begin(), v12.begin()),
       p1q2.size(),
       Kernel12({p0q2.AsVec64(), s02, z02, p1q1.AsVec64(), s11, xyzz11,
                 inP.halfedge_, inQ.halfedge_, inP.vertPos_, forward, p1q2}));
@@ -578,6 +578,9 @@ Boolean3::Boolean3(const Manifold::Impl &inP, const Manifold::Impl &inQ,
       Intersect12(inQ, inP, s20, p2q0, s11, p1q1, z20, xyzz11, p2q1_, false);
   PRINT("x21 size = " << x21_.size());
 
+  if (x12_.size() + x21_.size() >= std::numeric_limits<int>::max())
+    throw std::out_of_range("mesh too large");
+
   Vec<int> p0 = p0q2.Copy(false);
   p0q2.Resize(0);
   Vec<int> q0 = p2q0.Copy(true);

src/manifold/src/boolean_result.cpp

@@ -103,6 +103,10 @@ std::tuple<Vec<int>, Vec<int>> SizeOutput(
     bool invertQ) {
   ZoneScoped;
   Vec<int> sidesPerFacePQ(inP.NumTri() + inQ.NumTri(), 0);
+  // note: numFaceR <= facePQ2R.size() = sidesPerFacePQ.size() + 1
+  if (sidesPerFacePQ.size() + 1 >= std::numeric_limits<int>::max())
+    throw std::out_of_range("boolean result too large");
+
   auto sidesPerFaceP = sidesPerFacePQ.view(0, inP.NumTri());
   auto sidesPerFaceQ = sidesPerFacePQ.view(inP.NumTri(), inQ.NumTri());
 
@@ -175,7 +179,7 @@ void AddNewEdgeVerts(
   // edge. The direction and duplicity are given by i12, while v12R remaps to
   // the output vert index. When forward is false, all is reversed.
   auto process = [&](std::function<void(size_t)> lock,
-                     std::function<void(size_t)> unlock, int i) {
+                     std::function<void(size_t)> unlock, size_t i) {
     const int edgeP = p1q2.Get(i, !forward);
     const int faceQ = p1q2.Get(i, forward);
     const int vert = v12R[i];
@@ -218,17 +222,17 @@ void AddNewEdgeVerts(
         [&](size_t hash) { mutexes[hash % mutexes.size()].unlock(); },
         std::placeholders::_1);
     tbb::parallel_for(
-        tbb::blocked_range<int>(0, p1q2.size(), 32),
-        [&](const tbb::blocked_range<int> &range) {
-          for (int i = range.begin(); i != range.end(); i++) processFun(i);
+        tbb::blocked_range<size_t>(0_z, p1q2.size(), 32),
+        [&](const tbb::blocked_range<size_t> &range) {
+          for (size_t i = range.begin(); i != range.end(); i++) processFun(i);
         },
         ap);
     return;
   }
 #endif
   auto processFun = std::bind(
       process, [](size_t _) {}, [](size_t _) {}, std::placeholders::_1);
-  for (int i = 0; i < p1q2.size(); ++i) processFun(i);
+  for (size_t i = 0; i < p1q2.size(); ++i) processFun(i);
 }
 
 std::vector<Halfedge> PairUp(std::vector<EdgePos> &edgePos) {
@@ -239,15 +243,15 @@ std::vector<Halfedge> PairUp(std::vector<EdgePos> &edgePos) {
   // a heuristic.
   ASSERT(edgePos.size() % 2 == 0, topologyErr,
          "Non-manifold edge! Not an even number of points.");
-  int nEdges = edgePos.size() / 2;
+  size_t nEdges = edgePos.size() / 2;
   auto middle = std::partition(edgePos.begin(), edgePos.end(),
                                [](EdgePos x) { return x.isStart; });
   ASSERT(middle - edgePos.begin() == nEdges, topologyErr, "Non-manifold edge!");
   auto cmp = [](EdgePos a, EdgePos b) { return a.edgePos < b.edgePos; };
   std::stable_sort(edgePos.begin(), middle, cmp);
   std::stable_sort(middle, edgePos.end(), cmp);
   std::vector<Halfedge> edges;
-  for (int i = 0; i < nEdges; ++i)
+  for (size_t i = 0; i < nEdges; ++i)
     edges.push_back({edgePos[i].vert, edgePos[i + nEdges].vert, -1, -1});
   return edges;
 }
@@ -539,6 +543,11 @@ void CreateProperties(Manifold::Impl &outR, const Manifold::Impl &inP,
   std::vector<int> propMissIdx[2];
   propMissIdx[0].resize(inQ.NumPropVert(), -1);
   propMissIdx[1].resize(inP.NumPropVert(), -1);
+
+  if (static_cast<size_t>(outR.NumVert()) * static_cast<size_t>(numProp) >=
+      std::numeric_limits<int>::max())
+    throw std::out_of_range("too many vertices");
+
   outR.meshRelation_.properties.reserve(outR.NumVert() * numProp);
   int idx = 0;
 

src/manifold/src/edge_op.cpp

@@ -145,9 +145,9 @@ namespace manifold {
 void Manifold::Impl::SimplifyTopology() {
   if (!halfedge_.size()) return;
 
-  const int nbEdges = halfedge_.size();
+  const size_t nbEdges = halfedge_.size();
   auto policy = autoPolicy(nbEdges);
-  int numFlagged = 0;
+  size_t numFlagged = 0;
   Vec<uint8_t> bflags(nbEdges);
 
   // In the case of a very bad triangulation, it is possible to create pinched
@@ -191,8 +191,9 @@ void Manifold::Impl::SimplifyTopology() {
     ZoneScopedN("CollapseShortEdge");
     numFlagged = 0;
     ShortEdge se{halfedge_, vertPos_, precision_};
-    for_each_n(policy, countAt(0), nbEdges, [&](int i) { bflags[i] = se(i); });
-    for (int i = 0; i < nbEdges; ++i) {
+    for_each_n(policy, countAt(0_z), nbEdges,
+               [&](size_t i) { bflags[i] = se(i); });
+    for (size_t i = 0; i < nbEdges; ++i) {
       if (bflags[i]) {
         CollapseEdge(i, scratchBuffer);
         scratchBuffer.resize(0);
@@ -212,8 +213,9 @@ void Manifold::Impl::SimplifyTopology() {
     ZoneScopedN("CollapseFlaggedEdge");
     numFlagged = 0;
     FlagEdge se{halfedge_, meshRelation_.triRef};
-    for_each_n(policy, countAt(0), nbEdges, [&](int i) { bflags[i] = se(i); });
-    for (int i = 0; i < nbEdges; ++i) {
+    for_each_n(policy, countAt(0_z), nbEdges,
+               [&](size_t i) { bflags[i] = se(i); });
+    for (size_t i = 0; i < nbEdges; ++i) {
       if (bflags[i]) {
         CollapseEdge(i, scratchBuffer);
         scratchBuffer.resize(0);
@@ -233,11 +235,12 @@ void Manifold::Impl::SimplifyTopology() {
     ZoneScopedN("RecursiveEdgeSwap");
     numFlagged = 0;
     SwappableEdge se{halfedge_, vertPos_, faceNormal_, precision_};
-    for_each_n(policy, countAt(0), nbEdges, [&](int i) { bflags[i] = se(i); });
+    for_each_n(policy, countAt(0_z), nbEdges,
+               [&](size_t i) { bflags[i] = se(i); });
     std::vector<int> edgeSwapStack;
     std::vector<int> visited(halfedge_.size(), -1);
     int tag = 0;
-    for (int i = 0; i < nbEdges; ++i) {
+    for (size_t i = 0; i < nbEdges; ++i) {
       if (bflags[i]) {
         numFlagged++;
         tag++;

src/manifold/src/face_op.cpp

@@ -136,12 +136,12 @@ void Manifold::Impl::Face2Tri(const Vec<int>& faceEdge,
   tbb::task_group group;
   // map from face to triangle
   tbb::concurrent_unordered_map<int, std::vector<glm::ivec3>> results;
-  Vec<int> triCount(faceEdge.size());
+  Vec<size_t> triCount(faceEdge.size());
   triCount.back() = 0;
   // precompute number of triangles per face, and launch async tasks to
   // triangulate complex faces
-  for_each(autoPolicy(faceEdge.size()), countAt(0),
-           countAt(faceEdge.size() - 1), [&](int face) {
+  for_each(autoPolicy(faceEdge.size()), countAt(0_z),
+           countAt(faceEdge.size() - 1), [&](size_t face) {
              triCount[face] = faceEdge[face + 1] - faceEdge[face] - 2;
              ASSERT(triCount[face] >= 1, topologyErr,
                     "face has less than three edges.");
@@ -155,13 +155,15 @@ void Manifold::Impl::Face2Tri(const Vec<int>& faceEdge,
   group.wait();
   // prefix sum computation (assign unique index to each face) and preallocation
   exclusive_scan(autoPolicy(triCount.size()), triCount.begin(), triCount.end(),
-                 triCount.begin(), 0);
+                 triCount.begin(), 0_z);
+  if (triCount.back() >= std::numeric_limits<int>::max())
+    throw std::out_of_range("too many triangles");
   triVerts.resize(triCount.back());
   triNormal.resize(triCount.back());
   triRef.resize(triCount.back());
 
   auto processFace2 = std::bind(
-      processFace, [&](int face) { return std::move(results[face]); },
+      processFace, [&](size_t face) { return std::move(results[face]); },
       [&](int face, glm::ivec3 tri, glm::vec3 normal, TriRef r) {
         triVerts[triCount[face]] = tri;
         triNormal[triCount[face]] = normal;
@@ -170,7 +172,7 @@ void Manifold::Impl::Face2Tri(const Vec<int>& faceEdge,
       },
       std::placeholders::_1);
   // set triangles in parallel
-  for_each(autoPolicy(faceEdge.size()), countAt(0),
+  for_each(autoPolicy(faceEdge.size()), countAt(0_z),
            countAt(faceEdge.size() - 1), processFace2);
 #else
   triVerts.reserve(faceEdge.size());