Enum lowering and option to take snapshots.

docs/GettingStarted/debug.md

@@ -32,4 +32,7 @@ vast-opt --pass-pipeline="pipeline-string"
 
 ### Debug Pipeline
 
-With the `-vast-debug` option, you get more detailed crash reports. It shows MLIR operations when there's an error and provides current stack traces.
+With the `-vast-debug` option, you get more detailed crash reports. It shows MLIR operations when there's an error and provides current stack traces.
+
+Sometimes it is needed to examine the results of conversion steps more closely to discover what went wrong. `-vast-snapshot-at=pass1;...;passN` will instrument conversion pipeline to store a result of `passN` into a file after it is applied. Name of the file will be in the form of: `basename.pass_name`.
+Passing `"*"` in the string will result in output after every step.

docs/Tools/vast-front.md

@@ -44,6 +44,10 @@ Additional customization options include:
 - `-vast-disable-vast-verifier`
   - Skips verification of the produced VAST MLIR module.
 
+- `vast-snapshot-at="pass1;...;passN`
+  - After each pass that was specified as an option store MLIR into a file (format is `src.pass_name`).
+  - `"*"` stores snapshot after every conversion.
+
 ## Pipelines
 
-WIP pipelines documentation
+WIP pipelines documentation

include/vast/Dialect/HighLevel/HighLevelOps.td

@@ -634,7 +634,10 @@ class CastOp< string mnemonic, list< Trait > traits = [] >
     let assemblyFormat = "$value $kind attr-dict `:` type($value) `->` type($result)";
 }
 
-def ImplicitCastOp   : CastOp< "implicit_cast" >;
+def ImplicitCastOp   : CastOp< "implicit_cast" > {
+    let hasFolder = 1;
+}
+
 def CStyleCastOp     : CastOp< "cstyle_cast" >;
 def BuiltinBitCastOp : CastOp< "builtin_bitcast" >;
 

include/vast/Dialect/HighLevel/Passes.hpp

@@ -30,6 +30,8 @@ namespace vast::hl {
 
     std::unique_ptr< mlir::Pass > createSpliceTrailingScopes();
 
+    std::unique_ptr< mlir::Pass > createLowerEnumsPass();
+
 /// Generate the code for registering passes.
 #define GEN_PASS_REGISTRATION
 #include "vast/Dialect/HighLevel/Passes.h.inc"

include/vast/Dialect/HighLevel/Passes.td

@@ -120,4 +120,16 @@ def SpliceTrailingScopes : Pass<"vast-hl-splice-trailing-scopes", "mlir::ModuleO
   let constructor = "vast::hl::createSpliceTrailingScopes()";
 }
 
+def LowerEnums : Pass<"vast-hl-lower-enums", "mlir::ModuleOp"> {
+  let summary = "Lower high level representation of enums.";
+  let description = [{
+    Lower enum values and types into their underlying representations. Definition of enum
+    types are erased once conversion is finished.
+  }];
+
+  let constructor = "vast::hl::createLowerEnumsPass()";
+  let dependentDialects = [
+    "vast::hl::HighLevelDialect"
+  ];
+}
 #endif // VAST_DIALECT_HIGHLEVEL_PASSES_TD

include/vast/Frontend/Options.hpp

@@ -84,6 +84,8 @@ namespace vast::cc
         constexpr option_t simplify = "simplify";
         constexpr option_t canonicalize = "canonicalize";
 
+        constexpr option_t snapshot_at = "snapshot-at";
+
         llvm::Twine disable(string_ref pipeline_name);
 
         constexpr option_t show_locs = "show-locs";

include/vast/Frontend/Pipelines.hpp

@@ -48,7 +48,8 @@ namespace vast::cc {
     std::unique_ptr< vast_pipeline > setup_pipeline(
         pipeline_source src, target_dialect trg,
         mcontext_t &mctx,
-        const vast_args &vargs
+        const vast_args &vargs,
+        std::string snapshot_prefix
     );
 
 } // namespace vast::cc

include/vast/Util/Pipeline.hpp

@@ -1,4 +1,4 @@
 // Copyright (c) 2023-present, Trail of Bits, Inc.
 
 #pragma once
 
@@ -7,6 +7,7 @@
 VAST_RELAX_WARNINGS
 #include <llvm/ADT/DenseSet.h>
 #include <mlir/IR/DialectRegistry.h>
+#include <mlir/Pass/PassInstrumentation.h>
 #include <mlir/Pass/PassManager.h>
 #include <mlir/Pass/PassRegistry.h>
 #include <mlir/Pass/Pass.h>
@@ -65,9 +66,77 @@
 
         virtual void schedule(pipeline_step_ptr step) = 0;
 
+        void print_on_error(llvm::raw_ostream &os) {
+            enableIRPrinting(
+                [](auto *, auto *) { return false; }, // before
+                [](auto *, auto *) { return true; },  // after
+                false,                                // module scope
+                false,                                // after change
+                true,                                 // after failure
+                os
+            );
+        }
+
         llvm::DenseSet< pass_id_t > seen;
     };
 
+    struct with_snapshots : mlir::PassInstrumentation
+    {
+        using output_stream_ptr = std::shared_ptr< llvm::raw_pwrite_stream >;
+        using passes_t = std::vector< llvm::StringRef >;
+
+        std::string file_prefix;
+
+        with_snapshots(llvm::StringRef file_prefix)
+            : file_prefix(file_prefix.str())
+        {}
+
+        // We return `shared_ptr` in case we may want to keep the stream open for longer
+        // in some derived class. Should not make a difference, snapshoting will be expensive
+        // anyway.
+        virtual output_stream_ptr make_output_stream(mlir::Pass *pass) {
+            std::string name = file_prefix + "." + pass->getArgument().str();
+            std::error_code ec;
+            auto os = std::make_shared< llvm::raw_fd_ostream >(name, ec);
+            VAST_CHECK(!ec, "Cannot open file to store snapshot at, ec: {0}", ec.message());
+            return std::move(os);
+        }
+
+        virtual bool should_snapshot(mlir::Pass *pass) const = 0;
+
+        void runAfterPass(mlir::Pass *pass, operation op) override {
+            if (!should_snapshot(pass))
+                return;
+
+            auto os = make_output_stream(pass);
+            (*os) << *op;
+        }
+    };
+
+    struct snapshot_at_passes : with_snapshots {
+        using base = with_snapshots;
+
+        passes_t snapshot_at;
+
+        template< typename ... Args >
+        snapshot_at_passes(const passes_t &snapshot_at, Args && ... args)
+            : base(std::forward< Args >(args)...), snapshot_at(snapshot_at)
+        {}
+
+        virtual bool should_snapshot(mlir::Pass *pass) const override {
+            return std::ranges::count(snapshot_at, pass->getArgument());
+        }
+    };
+
+    struct snapshot_all : with_snapshots {
+        using base = with_snapshots;
+        using base::base;
+
+        bool should_snapshot(mlir::Pass *) const override {
+            return true;
+        }
+    };
+
 
     using pipeline_step_builder = std::function< pipeline_step_ptr(void) >;
 
@@ -104,9 +173,9 @@
         virtual ~pipeline_step() = default;
 
         virtual void schedule_on(pipeline_t &ppl) const = 0;
         virtual gap::generator< pipeline_step_ptr > substeps() const = 0;
 
         gap::generator< pipeline_step_ptr > dependencies() const;
 
         virtual string_ref name() const = 0;
 
@@ -128,7 +197,7 @@
 
         void schedule_on(pipeline_t &ppl) const override;
 
         gap::generator< pipeline_step_ptr > substeps() const override;
 
         string_ref name() const override;
 
@@ -162,7 +231,7 @@
 
         void schedule_on(pipeline_t &ppl) const override;
 
         gap::generator< pipeline_step_ptr > substeps() const override;
 
         string_ref name() const override;
 

lib/vast/Conversion/FromHL/EmitLazyRegions.cpp

@@ -1,4 +1,4 @@
 // Copyright (c) 2022-present, Trail of Bits, Inc.
 
 #include "vast/Util/Warnings.hpp"
 
@@ -133,6 +133,7 @@
         static conversion_target create_conversion_target(mcontext_t &context) {
             conversion_target target(context);
             target.addLegalDialect< vast::core::CoreDialect >();
+            target.markUnknownOpDynamicallyLegal([](auto){ return true; });
             return target;
         }
 

lib/vast/Conversion/FromHL/ToLLGEPs.cpp

@@ -1,4 +1,4 @@
 // Copyright (c) 2021-present, Trail of Bits, Inc.
 
 #include "vast/Dialect/HighLevel/Passes.hpp"
 
@@ -29,7 +29,12 @@
             logical_result matchAndRewrite(
                 op_t op, typename op_t::Adaptor ops, conversion_rewriter &rewriter
             ) const override {
-                auto parent_type = ops.getRecord().getType();
+                auto parent_type = [&] {
+                    auto type = ops.getRecord().getType();
+                    if (auto ptr = mlir::dyn_cast< hl::PointerType >(type))
+                        return ptr.getElementType();
+                    return type;
+                }();
 
                 auto mod = op->getParentOfType< vast_module >();
                 if (!mod) {

lib/vast/Dialect/HighLevel/HighLevelOps.cpp

@@ -1,4 +1,4 @@
 // Copyright (c) 2021-present, Trail of Bits, Inc.
 
 #include "vast/Util/Warnings.hpp"
 
@@ -61,6 +61,22 @@
         return {};
     }
 
+    namespace {
+
+        FoldResult fold_integral_cast(auto &self, auto adaptor) {
+            if (self.getResult().getType() == self.getValue().getType())
+                return self.getValue();
+            return {};
+        }
+
+    } // namespace
+
+    FoldResult ImplicitCastOp::fold(FoldAdaptor adaptor) {
+        if (getKind() == CastKind::IntegralCast)
+            return fold_integral_cast(*this, adaptor);
+        return {};
+    }
+
     FoldResult AddIOp::fold(FoldAdaptor adaptor) {
         return checked_int_arithmetic(getType(), adaptor,
             [] (const ap_sint &lhs, const ap_sint &rhs) -> std::optional< ap_sint > {

lib/vast/Dialect/HighLevel/Passes.cpp

@@ -47,6 +47,10 @@ namespace vast::hl::pipeline {
         return pass(hl::createUDEPass).depends_on(splice_trailing_scopes);
     }
 
+    static pipeline_step_ptr lower_enums() {
+        return pass(hl::createLowerEnumsPass).depends_on(desugar);
+    }
+
     pipeline_step_ptr simplify() {
         return compose("simplify",
             conv::pipeline::to_hlbi,
@@ -60,7 +64,7 @@ namespace vast::hl::pipeline {
     // stdtypes passes
     //
     pipeline_step_ptr stdtypes() {
-        return pass(hl::createHLLowerTypesPass).depends_on(desugar);
+        return pass(hl::createHLLowerTypesPass).depends_on(desugar, lower_enums);
     }
 
 } // namespace vast::hl::pipeline

lib/vast/Dialect/HighLevel/Transforms/CMakeLists.txt

@@ -5,6 +5,7 @@ add_vast_conversion_library(HighLevelTransforms
   HLLowerTypes.cpp
   DCE.cpp
   LowerElaboratedTypes.cpp
+  LowerEnums.cpp
   LowerTypeDefs.cpp
   SpliceTrailingScopes.cpp
   UDE.cpp

lib/vast/Dialect/HighLevel/Transforms/LowerEnums.cpp

@@ -0,0 +1,156 @@
+// Copyright (c) 2021-present, Trail of Bits, Inc.
+
+#include "vast/Dialect/HighLevel/Passes.hpp"
+
+VAST_RELAX_WARNINGS
+#include <mlir/Analysis/DataLayoutAnalysis.h>
+#include <mlir/IR/PatternMatch.h>
+#include <mlir/Transforms/DialectConversion.h>
+#include <mlir/Transforms/GreedyPatternRewriteDriver.h>
+
+#include <mlir/Transforms/Passes.h>
+#include <gap/core/overloads.hpp>
+VAST_UNRELAX_WARNINGS
+
+#include "PassesDetails.hpp"
+
+#include "vast/Dialect/HighLevel/HighLevelOps.hpp"
+#include "vast/Dialect/HighLevel/HighLevelUtils.hpp"
+#include "vast/Dialect/LowLevel/LowLevelOps.hpp"
+
+#include "vast/Conversion/TypeConverters/TypeConvertingPattern.hpp"
+
+#include "vast/Util/Common.hpp"
+#include "vast/Util/DialectConversion.hpp"
+#include "vast/Util/Symbols.hpp"
+#include "vast/Util/Terminator.hpp"
+
+namespace vast::hl {
+
+    struct enums_info {
+        using const_decl = hl::EnumConstantOp;
+        using enum_decl = hl::EnumDeclOp;
+
+        llvm::DenseMap< llvm::StringRef, const_decl > id_to_value;
+        llvm::DenseMap< llvm::StringRef, enum_decl > name_to_decl;
+
+        enums_info(vast_module mod) {
+            collect_enum_values(mod);
+        }
+
+        void collect_enum_values(vast_module root) {
+            auto walker = [&](operation op) {
+                if (auto decl = mlir::dyn_cast< const_decl >(op)) {
+                    VAST_ASSERT(!id_to_value.count(decl.getName()));
+                    id_to_value[decl.getName()] = decl;
+                } else if (auto decl = mlir::dyn_cast< enum_decl >(op)) {
+                    name_to_decl[decl.getName()] = decl;
+                }
+            };
+
+            root->walk(walker);
+        }
+    };
+
+    struct enum_type_converter
+        : conv::tc::identity_type_converter
+        , conv::tc::mixins< enum_type_converter >
+    {
+        mcontext_t &mctx;
+        const enums_info &info;
+
+        enum_type_converter(mcontext_t &mctx, const enums_info &info)
+            : mctx(mctx), info(info)
+        {
+            init();
+        }
+
+        auto convert_enum() {
+            return [&](hl::RecordType record) -> maybe_type_t {
+                // If not in the mapping it is not an `enum` (both `union` and `struct` use
+                // `record` as well.
+                auto name = record.getName();
+                auto it = info.name_to_decl.find(name);
+                if (it == info.name_to_decl.end())
+                    return {};
+
+                // Need to be a separate decl to avoid compiler thinking it is a `const`.
+                hl::EnumDeclOp decl = it->second;
+                auto type = decl.getType();
+                VAST_CHECK(type, "enum did not provide `getType()`, {0}", decl);
+                return type;
+            };
+        }
+
+        void init() {
+            addConversion(convert_enum());
+        }
+    };
+
+    struct type_rewriter : pattern_rewriter {
+        type_rewriter(mcontext_t *mctx) : pattern_rewriter(mctx) {}
+    };
+
+    using type_conversion_pattern = conv::tc::generic_type_converting_pattern< enum_type_converter >;
+
+    struct LowerEnumsPass : LowerEnumsBase< LowerEnumsPass > {
+        void runOnOperation() override {
+            auto enums = enums_info(this->getOperation());
+
+            // Investigate if we want these as patterns instead. I would be curious
+            // to see what's the runtime difference of repeating the traversal vs the
+            // bookkeeping the conversion needs to work.
+            // For now I am keeping this separate in case some random edge cases needs to be
+            // added (this is easier to debug),
+            // but eventually we probably should swap these to patterns.
+            replace_constants(enums);
+            replace_types(enums);
+
+            erase_enums(enums);
+
+            // TODO: Now the IR can be in a state where there are casts of `T -> T`. Should
+            //       we simply call canonizer?
+        }
+
+        void replace_constants(const enums_info &enums) {
+            auto replace_constants = [&](hl::EnumRefOp ref) {
+                auto name = ref.getValue();
+                auto it = enums.id_to_value.find(name);
+                VAST_ASSERT(it != enums.id_to_value.end());
+
+                auto bld = mlir::OpBuilder(ref);
+                hl::EnumConstantOp enum_constant = it->second;
+                auto value = enum_constant.getValue();
+
+                // Make the integer
+                operation i = bld.create< hl::ConstantOp >(ref.getLoc(), ref.getType(), value);
+                ref.replaceAllUsesWith(i);
+                ref->erase();
+            };
+
+            // First let's replace all constants
+            this->getOperation()->walk(replace_constants);
+        }
+
+        void replace_types(const enums_info &enums) {
+            auto tc = enum_type_converter(getContext(), enums);
+            auto pattern = type_conversion_pattern(tc, getContext());
+            auto replace_types = [&](operation root) {
+                type_rewriter bld(&getContext());
+                // We don't really care, failure only means that pattern was not applied,
+                // which is a valid result.
+                [[maybe_unused]] auto status = pattern.replace(root, bld);
+            };
+            this->getOperation()->walk(replace_types);
+        }
+
+        void erase_enums(const enums_info &enums) {
+            for (auto &[_, decl] : enums.name_to_decl)
+                decl->erase();
+        }
+    };
+} // namespace vast::hl
+
+std::unique_ptr< mlir::Pass > vast::hl::createLowerEnumsPass() {
+    return std::make_unique< vast::hl::LowerEnumsPass >();
+}