Implementation of a naive policy

.gitignore

@@ -48,3 +48,4 @@ deps/
 
 #Stats Files
 *stats.csv
+*.txt

include/pando-lib-galois/graphs/dist_local_csr.hpp

@@ -854,7 +854,7 @@ class DistLocalCSR {
     galois::HostLocalStorage<pando::Vector<VertexType>> pHV{};
     PANDO_CHECK_RETURN(pHV.initialize());
 
-    PANDO_CHECK_RETURN(galois::doAll(
+    PANDO_CHECK_RETURN(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
         partEdges, pHV,
         +[](HostLocalStorage<pando::Vector<pando::Vector<EdgeType>>> partEdges,
             pando::GlobalRef<pando::Vector<VertexType>> pHV) {

include/pando-lib-galois/import/ingest_rmat_el.hpp

@@ -92,7 +92,7 @@ ReturnType initializeELDACSR(pando::Array<char> filename, std::uint64_t numVerti
   galois::WaitGroup freeWaiter;
   PANDO_CHECK(freeWaiter.initialize(0));
   auto freeWGH = freeWaiter.getHandle();
-  galois::doAll(
+  galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
       freeWGH, perThreadRename, +[](galois::HashTable<std::uint64_t, std::uint64_t> hash) {
         hash.deinitialize();
       });
@@ -158,7 +158,7 @@ ReturnType initializeELDLCSR(pando::Array<char> filename, std::uint64_t numVerti
   galois::WaitGroup freeWaiter;
   PANDO_CHECK(freeWaiter.initialize(0));
   auto freeWGH = freeWaiter.getHandle();
-  galois::doAll(
+  galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
       freeWGH, perThreadRename, +[](galois::HashTable<std::uint64_t, std::uint64_t> hash) {
         hash.deinitialize();
       });
@@ -204,15 +204,16 @@ ReturnType initializeELDLCSR(pando::Array<char> filename, std::uint64_t numVerti
     }
   };
 
-  PANDO_CHECK(galois::doAll(generateVerticesState, pHV, generateVerticesPerHost));
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(generateVerticesState, pHV,
+                                                                   generateVerticesPerHost));
 
   auto [partEdges, renamePerHost] =
       internal::partitionEdgesParallely(pHV, std::move(localReadEdges), hostLocalV2PM);
 
-  galois::doAll(
+  galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
       partEdges, +[](pando::GlobalRef<pando::Vector<pando::Vector<ELEdge>>> edge_vectors) {
         pando::Vector<pando::Vector<ELEdge>> evs_tmp = edge_vectors;
-        galois::doAll(
+        galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
             evs_tmp, +[](pando::GlobalRef<pando::Vector<ELEdge>> src_ev) {
               pando::Vector<ELEdge> tmp = src_ev;
               std::sort(tmp.begin(), tmp.end());

include/pando-lib-galois/import/ingest_wmd_csv.hpp

@@ -104,7 +104,7 @@ galois::DistLocalCSR<VertexType, EdgeType> initializeWMDDLCSR(pando::Array<char>
   galois::WaitGroup freeWaiter;
   PANDO_CHECK(freeWaiter.initialize(0));
   auto freeWGH = freeWaiter.getHandle();
-  galois::doAll(
+  galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
       freeWGH, perThreadRename, +[](galois::HashTable<std::uint64_t, std::uint64_t> hash) {
         hash.deinitialize();
       });

include/pando-lib-galois/loops/do_all.hpp

@@ -53,9 +53,15 @@ enum SchedulerPolicy {
   UNSAFE_STRIPE,
   CORE_STRIPE,
   NODE_ONLY,
+  NAIVE,
 };
 
+#ifndef PANDO_SCHED
 constexpr SchedulerPolicy CURRENT_SCHEDULER_POLICY = SchedulerPolicy::RANDOM;
+#else
+constexpr SchedulerPolicy CURRENT_SCHEDULER_POLICY = PANDO_SCHED;
+#endif
+
 constexpr SchedulerPolicy EVENLY_PARITION_SCHEDULER_POLICY = SchedulerPolicy::CORE_STRIPE;
 
 template <enum SchedulerPolicy>
@@ -92,6 +98,11 @@ pando::Place schedulerImpl(pando::Place preferredLocality,
         pando::Place(preferredLocality.node, pando::anyPod, pando::CoreIndex(coreIdx, 0));
   } else if constexpr (Policy == NODE_ONLY) {
     preferredLocality = pando::Place(preferredLocality.node, pando::anyPod, pando::anyCore);
+  } else if constexpr (Policy == NAIVE) {
+    auto coreIdx = perCoreDist.getLocal()(perCoreRNG.getLocal());
+    assert(coreIdx < pando::getCoreDims().x);
+    preferredLocality =
+        pando::Place(pando::getCurrentNode(), pando::anyPod, pando::CoreIndex(coreIdx, 0));
   } else {
     PANDO_ABORT("SCHEDULER POLICY NOT IMPLEMENTED");
   }
@@ -180,7 +191,6 @@ class DoAll {
                              const L& localityFunc) {
     counter::HighResolutionCount<DOALL_TIMER_ENABLE> doAllTimer;
     doAllTimer.start();
-    LoopLocalSchedulerStruct<CURRENT_SCHEDULER_POLICY> loopLocal{};
     pando::Status err = pando::Status::Success;
 
     const auto end = range.end();
@@ -191,7 +201,7 @@ class DoAll {
 
     for (auto curr = range.begin(); curr != end; curr++) {
       // Required hack without workstealing
-      auto nodePlace = scheduler(localityFunc(s, *curr), loopLocal);
+      auto nodePlace = localityFunc(s, *curr);
       err = pando::executeOn(nodePlace, &notifyFunc<F, State, typename R::iterator>, func, s, curr,
                              wgh);
       if (err != pando::Status::Success) {
@@ -249,6 +259,37 @@ class DoAll {
     return err;
   }
 
+  template <SchedulerPolicy POLICY, typename State, typename R, typename F>
+  static pando::Status doAllExplicitPolicy(WaitGroup::HandleType wgh, State s, R& range,
+                                           const F& func) {
+    counter::HighResolutionCount<DOALL_TIMER_ENABLE> doAllTimer;
+    doAllTimer.start();
+    LoopLocalSchedulerStruct<POLICY> loopLocal{};
+    pando::Status err = pando::Status::Success;
+
+    const auto end = range.end();
+
+    std::uint64_t iter = 0;
+    std::uint64_t size = range.size();
+    wgh.add(size);
+
+    for (auto curr = range.begin(); curr != end; curr++) {
+      // Required hack without workstealing
+      auto nodePlace = schedulerImpl<POLICY>(localityOf(curr), loopLocal);
+      err = pando::executeOn(nodePlace, &notifyFunc<F, State, typename R::iterator>, func, s, curr,
+                             wgh);
+      if (err != pando::Status::Success) {
+        for (std::uint64_t i = iter; i < size; i++) {
+          wgh.done();
+        }
+        return err;
+      }
+      iter++;
+    }
+    counter::recordHighResolutionEvent(doAllCount, doAllTimer);
+    return err;
+  }
+
   /**
    * @brief This is the do_all loop from galois which takes a rang and lifts a function to it, and
    * adds a barrier afterwards.
@@ -291,6 +332,35 @@ class DoAll {
     return err;
   }
 
+  template <SchedulerPolicy POLICY, typename R, typename F>
+  static pando::Status doAllExplicitPolicy(WaitGroup::HandleType wgh, R& range, const F& func) {
+    counter::HighResolutionCount<DOALL_TIMER_ENABLE> doAllTimer;
+    doAllTimer.start();
+    LoopLocalSchedulerStruct<POLICY> loopLocal{};
+    pando::Status err = pando::Status::Success;
+
+    const auto end = range.end();
+
+    std::uint64_t iter = 0;
+    const std::uint64_t size = range.size();
+    wgh.add(size);
+
+    for (auto curr = range.begin(); curr != end; curr++) {
+      // Required hack without workstealing
+      auto nodePlace = schedulerImpl<POLICY>(localityOf(curr), loopLocal);
+      err = pando::executeOn(nodePlace, &notifyFunc<F, typename R::iterator>, func, curr, wgh);
+      if (err != pando::Status::Success) {
+        for (std::uint64_t i = iter; i < size; i++) {
+          wgh.done();
+        }
+        return err;
+      }
+      iter++;
+    }
+    counter::recordHighResolutionEvent(doAllCount, doAllTimer);
+    return err;
+  }
+
   /**
    * @brief This is the do_all loop from galois which takes a range and lifts a function to it, and
    * adds a barrier afterwards.
@@ -342,6 +412,20 @@ class DoAll {
     return err;
   }
 
+  template <SchedulerPolicy POLICY, typename State, typename R, typename F>
+  static pando::Status doAllExplicitPolicy(State s, R& range, const F& func) {
+    pando::Status err;
+    WaitGroup wg;
+    err = wg.initialize(0);
+    if (err != pando::Status::Success) {
+      return err;
+    }
+    doAllExplicitPolicy<POLICY, State, R, F>(wg.getHandle(), s, range, func);
+    err = wg.wait();
+    wg.deinitialize();
+    return err;
+  }
+
   /**
    * @brief This is the do_all loop from galois which takes a range and lifts a function to it, and
    * adds a barrier afterwards.
@@ -366,6 +450,21 @@ class DoAll {
     return err;
   }
 
+  template <SchedulerPolicy POLICY, typename R, typename F>
+  static pando::Status doAllExplicitPolicy(R& range, const F& func) {
+    pando::Status err;
+    WaitGroup wg;
+    err = wg.initialize(0);
+    if (err != pando::Status::Success) {
+      return err;
+    }
+    doAllExplicitPolicy<POLICY, R, F>(wg.getHandle(), range, func);
+
+    err = wg.wait();
+    wg.deinitialize();
+    return err;
+  }
+
   /**
    * @brief This is the do_all loop from galois which takes an integer number of work items and
    * lifts a function to it, and adds a barrier afterwards.  Work is spread evenly across all pxns
@@ -498,10 +597,18 @@ template <typename State, typename R, typename F>
 pando::Status doAll(WaitGroup::HandleType wgh, State s, R range, const F& func) {
   return DoAll::doAll<State, R, F>(wgh, s, range, func);
 }
+template <SchedulerPolicy POLICY, typename State, typename R, typename F>
+pando::Status doAllExplicitPolicy(WaitGroup::HandleType wgh, State s, R range, const F& func) {
+  return DoAll::doAllExplicitPolicy<POLICY, State, R, F>(wgh, s, range, func);
+}
 template <typename R, typename F>
 pando::Status doAll(WaitGroup::HandleType wgh, R range, const F& func) {
   return DoAll::doAll<R, F>(wgh, range, func);
 }
+template <SchedulerPolicy POLICY, typename R, typename F>
+pando::Status doAllExplicitPolicy(WaitGroup::HandleType wgh, R range, const F& func) {
+  return DoAll::doAllExplicitPolicy<POLICY, R, F>(wgh, range, func);
+}
 template <typename State, typename R, typename F, typename L>
 pando::Status doAll(State s, R range, const F& func, const L& localityFunc) {
   return DoAll::doAll<State, R, F, L>(s, range, func, localityFunc);
@@ -510,11 +617,18 @@ template <typename State, typename R, typename F>
 pando::Status doAll(State s, R range, const F& func) {
   return DoAll::doAll<State, R, F>(s, range, func);
 }
+template <SchedulerPolicy POLICY, typename State, typename R, typename F>
+pando::Status doAllExplicitPolicy(State s, R range, const F& func) {
+  return DoAll::doAllExplicitPolicy<POLICY, State, R, F>(s, range, func);
+}
 template <typename R, typename F>
 pando::Status doAll(R range, const F& func) {
   return DoAll::doAll<R, F>(range, func);
 }
-
+template <SchedulerPolicy POLICY, typename R, typename F>
+pando::Status doAllExplicitPolicy(R range, const F& func) {
+  return DoAll::doAllExplicitPolicy<POLICY, R, F>(range, func);
+}
 template <typename State, typename F>
 pando::Status doAllEvenlyPartition(WaitGroup::HandleType wgh, State s, uint64_t workItems,
                                    const F& func) {

microbench/bfs/include/pando-bfs-galois/sssp.hpp

@@ -134,7 +134,7 @@ pando::Status SSSP_DLCSR(
   state.dist = 0;
 
 #ifdef PANDO_STAT_TRACE_ENABLE
-  PANDO_CHECK(galois::doAll(
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
       wgh, phbfs, +[](pando::Vector<typename G::VertexTopologyID>) {
         PANDO_MEM_STAT_NEW_KERNEL("BFS Start");
       }));
@@ -168,7 +168,7 @@ pando::Status SSSP_DLCSR(
   }
 
 #ifdef PANDO_STAT_TRACE_ENABLE
-  PANDO_CHECK(galois::doAll(
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
       wgh, phbfs, +[](pando::Vector<typename G::VertexTopologyID>) {
         PANDO_MEM_STAT_NEW_KERNEL("BFS END");
       }));
@@ -311,7 +311,7 @@ pando::Status SSSPMDLCSR(G& graph, std::uint64_t src, HostLocalStorage<MDWorkLis
 #endif
 
 #ifdef PANDO_STAT_TRACE_ENABLE
-  PANDO_CHECK(galois::doAll(
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
       wgh, toRead, +[](MDWorkList<G>) {
         PANDO_MEM_STAT_NEW_KERNEL("BFS Start");
       }));
@@ -360,7 +360,7 @@ pando::Status SSSPMDLCSR(G& graph, std::uint64_t src, HostLocalStorage<MDWorkLis
   }
 
 #ifdef PANDO_STAT_TRACE_ENABLE
-  PANDO_CHECK(galois::doAll(
+  PANDO_CHECK(galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
       wgh, toRead, +[](MDWorkList<G>) {
         PANDO_MEM_STAT_NEW_KERNEL("BFS END");
       }));

scripts/run-drv.sh

@@ -22,7 +22,7 @@ usage: preprun -n <n> prog [program args]
 "
 }
 
-HOST_THREADS="${HOST_THREADS:-16}"
+HOST_THREADS="${HOST_THREADS:-1}"
 HOSTS="${HOSTS:-1}"
 PROCS="${PROCS:-2}"
 CORES="${CORES:-8}"
@@ -72,7 +72,7 @@ shift
 ${DBG} sst -n ${HOST_THREADS} \
     "${LAUNCH_SCRIPT}" -- \
     --with-command-processor="${PROG}" \
-    --num-pxn=${PROCS} \
+    --num-pxn=${HOSTS} \
     --pod-cores=${CORES} \
     --core-threads=${HARTS} \
     --drvx-stack-in-l1sp \

src/ingest_rmat_el.cpp

@@ -92,7 +92,7 @@ pando::Vector<pando::Vector<galois::ELEdge>> galois::reduceLocalEdges(
     }
   }
 
-  galois::doAll(
+  galois::doAllExplicitPolicy<SchedulerPolicy::RANDOM>(
       reducedEL, +[](pando::Vector<galois::ELEdge> src_ev) {
         std::sort(src_ev.begin(), src_ev.end());
       });