finally fixed bugs?

TODO.md

@@ -0,0 +1,3 @@
+- remapping is very brittle as it relies on `get_neighbour_colours`
+  - this function depends on neighbour container (colour indices, as well as sorting) which depends on `multiset_hash` flag and the WL feature generation algorithm being used
+  - `bulk_pruner` also uses this function but assumes WL feature generation only, i.e. each tuple only gives one node

include/feature_generation/feature_generators/iwl.hpp

@@ -34,7 +34,8 @@ namespace feature_generation {
     void collect_impl(const std::vector<graph::Graph> &graphs) override;
     void refine(const std::shared_ptr<graph::Graph> &graph,
                 std::vector<int> &colours,
-                std::vector<int> &colours_tmp);
+                std::vector<int> &colours_tmp,
+                int iteration);
   };
 }  // namespace feature_generation
 

include/feature_generation/feature_generators/kwl2.hpp

@@ -31,7 +31,7 @@ namespace feature_generation {
     Embedding embed(const std::shared_ptr<graph::Graph> &graph) override;
 
    protected:
-    std::vector<int> get_neighbour_colour_indices(const std::vector<int> &colours);
+    std::vector<std::pair<int, int>> get_neighbour_colours(const std::vector<int> &colours);
     inline int get_initial_colour(int index,
                                   int u,
                                   int v,
@@ -40,7 +40,8 @@ namespace feature_generation {
     void collect_impl(const std::vector<graph::Graph> &graphs) override;
     void refine(const std::shared_ptr<graph::Graph> &graph,
                 std::vector<int> &colours,
-                std::vector<int> &colours_tmp);
+                std::vector<int> &colours_tmp,
+                int iteration);
   };
 }  // namespace feature_generation
 

include/feature_generation/feature_generators/lwl2.hpp

@@ -33,7 +33,8 @@ namespace feature_generation {
     void refine(const std::shared_ptr<graph::Graph> &graph,
                 std::vector<std::set<int>> &pair_to_neighbours,
                 std::vector<int> &colours,
-                std::vector<int> &colours_tmp);
+                std::vector<int> &colours_tmp,
+                int iteration);
   };
 }  // namespace feature_generation
 

include/feature_generation/feature_generators/wl.hpp

@@ -28,11 +28,12 @@ namespace feature_generation {
     Embedding embed(const std::shared_ptr<graph::Graph> &graph) override;
 
    protected:
-    std::vector<int> get_neighbour_colour_indices(const std::vector<int> &colours);
+    std::vector<std::pair<int, int>> get_neighbour_colours(const std::vector<int> &colours);
     void collect_impl(const std::vector<graph::Graph> &graphs) override;
     void refine(const std::shared_ptr<graph::Graph> &graph,
                 std::vector<int> &colours,
-                std::vector<int> &colours_tmp);
+                std::vector<int> &colours_tmp,
+                int iteration);
   };
 }  // namespace feature_generation
 

include/feature_generation/features.hpp

@@ -45,7 +45,8 @@ class int_vector_hasher {
 
 namespace feature_generation {
   using Embedding = std::vector<int>;
-  using ColourHash = std::unordered_map<std::vector<int>, int, int_vector_hasher>;
+  using VecColourHash = std::vector<std::unordered_map<std::vector<int>, int, int_vector_hasher>>;
+  using StrColourHash = std::vector<std::unordered_map<std::string, int>>;
 
   class Features {
    protected:
@@ -58,7 +59,7 @@ namespace feature_generation {
     bool multiset_hash;
 
     // colouring [saved]
-    ColourHash colour_hash;
+    VecColourHash colour_hash;
     std::unordered_map<int, int> colour_to_layer;
     std::vector<std::set<int>> layer_to_colours;
 
@@ -72,7 +73,6 @@ namespace feature_generation {
     bool collected;
     bool pruned;
     bool collecting;
-    int cur_collecting_layer;
     std::shared_ptr<NeighbourContainer> neighbour_container;
 
     // runtime statistics; int is faster than long but could cause overflow
@@ -90,12 +90,16 @@ namespace feature_generation {
     std::vector<graph::Graph> convert_to_graphs(const data::Dataset dataset);
 
     // get hashed colour if it exists, and constructs it if it doesn't
-    int get_colour_hash(const std::vector<int> &colour);
+    int get_colour_hash(const std::vector<int> &colour, const int iteration);
 
     // reformat colour hash based on colours to throw out
-    void init_layer_to_colours();
+    VecColourHash new_colour_hash() const;
+    std::vector<std::set<int>> new_layer_to_colours() const;
     std::map<int, int> remap_colour_hash(const std::set<int> &to_prune);
-    virtual std::vector<int> get_neighbour_colour_indices(const std::vector<int> &colours) = 0;
+
+    // TODO redesign this with neighbour container
+    virtual std::vector<std::pair<int, int>>
+    get_neighbour_colours(const std::vector<int> &colours) = 0;
     std::vector<int> remap_neighbour_colours(const std::vector<int> &colours,
                                              const std::map<int, int> &remap);
 
@@ -128,6 +132,8 @@ namespace feature_generation {
     Embedding embed_state(const planning::State &state);
     virtual Embedding embed(const std::shared_ptr<graph::Graph> &graph) = 0;
 
+    void add_colour_to_x(int colour, int iteration, std::vector<int> &x);
+
     /* Pruning functions */
 
     void prune_this_iteration(int iteration,
@@ -161,7 +167,7 @@ namespace feature_generation {
     std::set<int> get_iteration_colours(int iteration) const {
       return layer_to_colours.at(iteration);
     }
-    ColourHash get_colour_hash() { return colour_hash; }
+    VecColourHash get_colour_hash() { return colour_hash; }
 
     /* Util functions */
 
@@ -173,11 +179,11 @@ namespace feature_generation {
     void set_problem(const planning::Problem &problem);
 
     // conversion between vectors and strings
-    ColourHash str_to_int_colour_hash(std::unordered_map<std::string, int> str_colour_hash) const;
-    std::unordered_map<std::string, int> int_to_str_colour_hash(ColourHash int_colour_hash) const;
+    VecColourHash str_to_int_colour_hash(StrColourHash str_colour_hash) const;
+    StrColourHash int_to_str_colour_hash(VecColourHash int_colour_hash) const;
 
     // statistics functions
-    int get_n_features() const { return colour_hash.size(); }
+    int get_n_features() const;
     std::vector<long> get_seen_counts() const { return seen_colour_statistics[1]; };
     std::vector<long> get_unseen_counts() const { return seen_colour_statistics[0]; };
     int get_n_seen_graphs() const { return n_seen_graphs; }

setup.py

@@ -7,11 +7,14 @@
 # Read version from wlplan/__version__.py file
 exec(open("wlplan/__version__.py").read())
 
+# Debug mode
+_DEBUG = False
+
 # Compiler flags
-COMPILER_FLAGS = [
-    "-O3",
-    # "-DDEBUGMODE",
-]
+if _DEBUG:
+    COMPILER_FLAGS = ["-O0", "-g", "-DDEBUGMODE"]
+else:
+    COMPILER_FLAGS = ["-O3"]
 
 files = [glob("src/*.cpp"), glob("src/**/*.cpp"), glob("src/**/**/*.cpp")]
 

src/feature_generation/feature_generators/ccwl.cpp

@@ -29,7 +29,7 @@ namespace feature_generation {
     // To change this to max, we just need to replace += occurrences with std::max.
 
     /* 1. Initialise embedding before pruning */
-    int categorical_size = colour_hash.size();
+    int categorical_size = get_n_features();
     Embedding x0(categorical_size * 2, 0);
 
     /* 2. Set up memory for WL updates */
@@ -41,7 +41,7 @@ namespace feature_generation {
     int col;
     int is_seen_colour;
     for (int node_i = 0; node_i < n_nodes; node_i++) {
-      col = get_colour_hash({graph->nodes[node_i]});
+      col = get_colour_hash({graph->nodes[node_i]}, 0);
       colours[node_i] = col;
       is_seen_colour = (col != UNSEEN_COLOUR);  // prevent branch prediction
       seen_colour_statistics[is_seen_colour][0]++;
@@ -53,7 +53,7 @@ namespace feature_generation {
 
     /* 4. Main WL loop */
     for (int itr = 1; itr < iterations + 1; itr++) {
-      refine(graph, colours, colours_tmp);
+      refine(graph, colours, colours_tmp, itr);
       for (int node_i = 0; node_i < n_nodes; node_i++) {
         col = colours[node_i];
         is_seen_colour = (col != UNSEEN_COLOUR);  // prevent branch prediction

src/feature_generation/feature_generators/iwl.cpp

@@ -29,7 +29,8 @@ namespace feature_generation {
 
   void IWLFeatures::refine(const std::shared_ptr<graph::Graph> &graph,
                            std::vector<int> &colours,
-                           std::vector<int> &colours_tmp) {
+                           std::vector<int> &colours_tmp,
+                           int iteration) {
     // memory for storing string and hashed int representation of colours
     std::vector<int> new_colour;
     std::vector<int> neighbour_vector;
@@ -61,7 +62,7 @@ namespace feature_generation {
       new_colour.insert(new_colour.end(), neighbour_vector.begin(), neighbour_vector.end());
 
       // hash seen colours
-      new_colour_compressed = get_colour_hash(new_colour);
+      new_colour_compressed = get_colour_hash(new_colour, iteration);
 
     end_of_iteration:
       colours_tmp[u] = new_colour_compressed;
@@ -93,17 +94,15 @@ namespace feature_generation {
         graph->change_node_colour(node_i, INDIVIDUALISE_COLOUR);
 
         // init colours
-        cur_collecting_layer = 0;
         for (int u = 0; u < n_nodes; u++) {
-          int col = get_colour_hash({graph->nodes[u]});
+          int col = get_colour_hash({graph->nodes[u]}, 0);
           colours[u] = col;
           seen_initial_colours.insert(col);
         }
 
         // main WL loop
         for (int iteration = 1; iteration < iterations + 1; iteration++) {
-          cur_collecting_layer = iteration;
-          refine(graph, colours, colours_tmp);
+          refine(graph, colours, colours_tmp, iteration);
         }
 
         // reset node colour
@@ -119,7 +118,7 @@ namespace feature_generation {
     }
 
     /* 1. Initialise embedding */
-    Embedding x0(colour_hash.size(), 0);
+    Embedding x0(get_n_features(), 0);
 
     /* 2. Set up memory for WL updates */
     int n_nodes = graph->nodes.size();
@@ -134,22 +133,17 @@ namespace feature_generation {
       graph->change_node_colour(node_i, INDIVIDUALISE_COLOUR);
 
       /* 3. Compute initial colours */
-      int is_seen_colour;
       for (int u = 0; u < n_nodes; u++) {
-        int col = get_colour_hash({graph->nodes[u]});
+        int col = get_colour_hash({graph->nodes[u]}, 0);
         colours[u] = col;
-        is_seen_colour = (col != UNSEEN_COLOUR);  // prevent branch prediction
-        seen_colour_statistics[is_seen_colour][0]++;
-        x0[col] += is_seen_colour;
+        add_colour_to_x(col, 0, x0);
       }
 
       /* 4. Main WL loop */
       for (int itr = 1; itr < iterations + 1; itr++) {
-        refine(graph, colours, colours_tmp);
+        refine(graph, colours, colours_tmp, itr);
         for (const int col : colours) {
-          is_seen_colour = (col != UNSEEN_COLOUR);  // prevent branch prediction
-          seen_colour_statistics[is_seen_colour][itr]++;
-        x0[col] += is_seen_colour;
+          add_colour_to_x(col, itr, x0);
         }
       }
 

src/feature_generation/feature_generators/kwl2.cpp

@@ -26,10 +26,11 @@ namespace feature_generation {
 
   KWL2Features::KWL2Features(const std::string &filename) : Features(filename) {}
 
-  std::vector<int> KWL2Features::get_neighbour_colour_indices(const std::vector<int> &colours) {
+  std::vector<std::pair<int, int>>
+  KWL2Features::get_neighbour_colours(const std::vector<int> &colours) {
     std::cout << "not implemented yet" << std::endl;
     exit(-1);
-    return std::vector<int>();
+    return std::vector<std::pair<int, int>>();
   }
 
   int kwl2_pair_to_index_map(int n, int i, int j) {
@@ -41,7 +42,8 @@ namespace feature_generation {
 
   void KWL2Features::refine(const std::shared_ptr<graph::Graph> &graph,
                             std::vector<int> &colours,
-                            std::vector<int> &colours_tmp) {
+                            std::vector<int> &colours_tmp,
+                            int iteration) {
     // memory for storing string and hashed int representation of colours
     std::vector<int> new_colour;
     std::vector<int> neighbour_vector;
@@ -78,7 +80,7 @@ namespace feature_generation {
         new_colour.insert(new_colour.end(), neighbour_vector.begin(), neighbour_vector.end());
 
         // hash seen colours
-        new_colour_compressed = get_colour_hash(new_colour);
+        new_colour_compressed = get_colour_hash(new_colour, iteration);
 
       end_of_iteration:
         colours_tmp[index] = new_colour_compressed;
@@ -109,7 +111,7 @@ namespace feature_generation {
     int u_col = graph->nodes[u];
     int v_col = graph->nodes[v];
     int e_col = pair_to_edge_label[index];
-    int col = get_colour_hash({u_col, v_col, e_col});
+    int col = get_colour_hash({u_col, v_col, e_col}, 0);
     return col;
   }
 
@@ -136,7 +138,6 @@ namespace feature_generation {
       std::vector<int> pair_to_edge_label = get_kwl2_pair_to_edge_label(graph);
 
       // init colours
-      cur_collecting_layer = 0;
       for (int u = 0; u < n_nodes; u++) {
         for (int v = 0; v < n_nodes; v++) {
           int index = kwl2_pair_to_index_map(n_nodes, u, v);
@@ -148,8 +149,7 @@ namespace feature_generation {
 
       // main WL loop
       for (int iteration = 1; iteration < iterations + 1; iteration++) {
-        cur_collecting_layer = iteration;
-        refine(graph, colours, colours_tmp);
+        refine(graph, colours, colours_tmp, iteration);
       }
     }
   }
@@ -161,7 +161,7 @@ namespace feature_generation {
     }
 
     /* 1. Initialise embedding before pruning */
-    Embedding x0(colour_hash.size(), 0);
+    Embedding x0(get_n_features(), 0);
 
     /* 2. Set up memory for WL updates */
     int n_nodes = graph->nodes.size();
@@ -172,25 +172,20 @@ namespace feature_generation {
     std::vector<int> pair_to_edge_label = get_kwl2_pair_to_edge_label(graph);
 
     /* 3. Compute initial colours */
-    int is_seen_colour;
     for (int u = 0; u < n_nodes; u++) {
       for (int v = 0; v < n_nodes; v++) {
         int index = kwl2_pair_to_index_map(n_nodes, u, v);
         int col = get_initial_colour(index, u, v, graph, pair_to_edge_label);
         colours[index] = col;
-        is_seen_colour = (col != UNSEEN_COLOUR);  // prevent branch prediction
-        seen_colour_statistics[is_seen_colour][0]++;
-        x0[col] += is_seen_colour;
+        add_colour_to_x(col, 0, x0);
       }
     }
 
     /* 4. Main WL loop */
     for (int itr = 1; itr < iterations + 1; itr++) {
-      refine(graph, colours, colours_tmp);
+      refine(graph, colours, colours_tmp, itr);
       for (const int col : colours) {
-        is_seen_colour = (col != UNSEEN_COLOUR);  // prevent branch prediction
-        seen_colour_statistics[is_seen_colour][itr]++;
-        x0[col] += is_seen_colour;
+        add_colour_to_x(col, itr, x0);
       }
     }