Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ComputeEngine Property for choosing Engine #2849

Merged
merged 1 commit into from
Jan 2, 2025
Merged

Add ComputeEngine Property for choosing Engine #2849

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions api/ccapi/include/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,15 @@ enum class ExecutionMode {
VALIDATE /** Validate mode, label is necessary */
};

/**
* @brief Enumeration of layer compute engine
*/
enum LayerComputeEngine {
CPU, /**< CPU as the compute engine */
GPU, /**< GPU as the compute engine */
QNN, /**< QNN as the compute engine */
};

/**
* @brief Get the version of NNTrainer
*/
Expand Down
63 changes: 24 additions & 39 deletions api/ccapi/include/layer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,14 +114,6 @@ enum LayerType {
LAYER_UNKNOWN = ML_TRAIN_LAYER_TYPE_UNKNOWN /**< Unknown */
};

/**
* @brief Enumeration of layer compute engine
*/
enum LayerComputeEngine {
CPU, /**< CPU as the compute engine */
GPU, /**< GPU as the compute engine */
};

/**
* @class Layer Base class for layers
* @brief Base class for all layers
Expand Down Expand Up @@ -261,16 +253,14 @@ class Layer {
*/
std::unique_ptr<Layer>
createLayer(const LayerType &type,
const std::vector<std::string> &properties = {},
const LayerComputeEngine &compute_engine = LayerComputeEngine::CPU);
const std::vector<std::string> &properties = {});

/**
* @brief Factory creator with constructor for layer
*/
std::unique_ptr<Layer>
createLayer(const std::string &type,
const std::vector<std::string> &properties = {},
const LayerComputeEngine &compute_engine = LayerComputeEngine::CPU);
const std::vector<std::string> &properties = {});

/**
* @brief General Layer Factory function to register Layer
Expand Down Expand Up @@ -343,37 +333,35 @@ DivideLayer(const std::vector<std::string> &properties = {}) {
/**
* @brief Helper function to create fully connected layer
*/
inline std::unique_ptr<Layer> FullyConnected(
const std::vector<std::string> &properties = {},
const LayerComputeEngine &compute_engine = LayerComputeEngine::CPU) {
return createLayer(LayerType::LAYER_FC, properties, compute_engine);
inline std::unique_ptr<Layer>
FullyConnected(const std::vector<std::string> &properties = {}) {
return createLayer(LayerType::LAYER_FC, properties);
}

/**
* @brief Helper function to create Swiglu layer
*/
inline std::unique_ptr<Layer>
Swiglu(const std::vector<std::string> &properties = {},
const LayerComputeEngine &compute_engine = LayerComputeEngine::CPU) {
return createLayer(LayerType::LAYER_SWIGLU, properties, compute_engine);
Swiglu(const std::vector<std::string> &properties = {}) {
return createLayer(LayerType::LAYER_SWIGLU, properties);
}

/**
* @brief Helper function to create RMS normalization layer for GPU
*/
inline std::unique_ptr<Layer>
RMSNormCl(const std::vector<std::string> &properties = {},
const LayerComputeEngine &compute_engine = LayerComputeEngine::GPU) {
return createLayer(LayerType::LAYER_RMSNORM, properties, compute_engine);
}
// /**
// * @brief Helper function to create RMS normalization layer for GPU
// */
// inline std::unique_ptr<Layer>
// RMSNormCl(const std::vector<std::string> &properties = {},
// const LayerComputeEngine &compute_engine = LayerComputeEngine::GPU)
// {
// return createLayer(LayerType::LAYER_RMSNORM, properties, compute_engine);
// }
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's remove it

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it might be better to remove this after evaluating the cl_context.


/**
* @brief Helper function to create Transpose layer
*/
inline std::unique_ptr<Layer>
Transpose(const std::vector<std::string> &properties = {},
const LayerComputeEngine &compute_engine = LayerComputeEngine::CPU) {
return createLayer(LayerType::LAYER_TRANSPOSE, properties, compute_engine);
Transpose(const std::vector<std::string> &properties = {}) {
return createLayer(LayerType::LAYER_TRANSPOSE, properties);
}

/**
Expand Down Expand Up @@ -428,27 +416,24 @@ Flatten(const std::vector<std::string> &properties = {}) {
* @brief Helper function to create reshape layer
*/
inline std::unique_ptr<Layer>
Reshape(const std::vector<std::string> &properties = {},
const LayerComputeEngine &compute_engine = LayerComputeEngine::CPU) {
return createLayer(LayerType::LAYER_RESHAPE, properties, compute_engine);
Reshape(const std::vector<std::string> &properties = {}) {
return createLayer(LayerType::LAYER_RESHAPE, properties);
}

/**
* @brief Helper function to create addition layer
*/
inline std::unique_ptr<Layer>
Addition(const std::vector<std::string> &properties = {},
const LayerComputeEngine &compute_engine = LayerComputeEngine::CPU) {
return createLayer(LayerType::LAYER_ADDITION, properties, compute_engine);
Addition(const std::vector<std::string> &properties = {}) {
return createLayer(LayerType::LAYER_ADDITION, properties);
}

/**
* @brief Helper function to create concat layer
*/
inline std::unique_ptr<Layer>
Concat(const std::vector<std::string> &properties = {},
const LayerComputeEngine &compute_engine = LayerComputeEngine::CPU) {
return createLayer(LayerType::LAYER_CONCAT, properties, compute_engine);
Concat(const std::vector<std::string> &properties = {}) {
return createLayer(LayerType::LAYER_CONCAT, properties);
}

/**
Expand Down
10 changes: 4 additions & 6 deletions api/ccapi/src/factory.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,16 @@ namespace ml {
namespace train {

std::unique_ptr<Layer> createLayer(const LayerType &type,
const std::vector<std::string> &properties,
const LayerComputeEngine &compute_engine) {
return nntrainer::createLayerNode(type, properties, compute_engine);
const std::vector<std::string> &properties) {
return nntrainer::createLayerNode(type, properties);
}

/**
* @brief Factory creator with constructor for layer
*/
std::unique_ptr<Layer> createLayer(const std::string &type,
const std::vector<std::string> &properties,
const LayerComputeEngine &compute_engine) {
return nntrainer::createLayerNode(type, properties, compute_engine);
const std::vector<std::string> &properties) {
return nntrainer::createLayerNode(type, properties);
}

std::unique_ptr<Optimizer>
Expand Down
24 changes: 23 additions & 1 deletion nntrainer/layers/common_properties.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include <string>

#include <base_properties.h>
#include <common.h>
#include <connection.h>
#include <tensor.h>
#include <tensor_wrap_specs.h>
Expand Down Expand Up @@ -945,12 +946,33 @@ struct ActivationTypeInfo {
* @brief Activation Enumeration Information
*
*/
class Activation final : public EnumProperty<ActivationTypeInfo> {
class Activation final
: public EnumProperty<nntrainer::props::ActivationTypeInfo> {
public:
using prop_tag = enum_class_prop_tag;
static constexpr const char *key = "activation";
};

/**
* @brief Enumeration of Run Engine type
*/
struct ComputeEngineTypeInfo {
using Enum = ml::train::LayerComputeEngine;
static constexpr std::initializer_list<Enum> EnumList = {Enum::CPU, Enum::GPU,
Enum::QNN};
static constexpr const char *EnumStr[] = {"cpu", "gpu", "qnn"};
};

/**
* @brief ComputeEngine Enumeration Information
*
*/
class ComputeEngine final : public EnumProperty<ComputeEngineTypeInfo> {
public:
using prop_tag = enum_class_prop_tag;
static constexpr const char *key = "engine";
};

/**
* @brief HiddenStateActivation Enumeration Information
*
Expand Down
75 changes: 53 additions & 22 deletions nntrainer/layers/layer_node.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,20 +130,49 @@ class SharedFrom : public Name {
*/
LayerNode::~LayerNode() = default;

/**
* @brief get the compute engine property from property string vector
* : default is CPU
* @return LayerComputeEngine Enum : CPU, GPU, QNN
*
*/
ml::train::LayerComputeEngine
getComputeEngine(const std::vector<std::string> &props) {
Comment on lines +139 to +140
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function prototype for getComputeEngine is missing in header. Isn't it required to be declared in header?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for your comment. This getComputeEngine function will be removed in PR #2849.
It moved and renamed as "parseComputeEngine"

for (auto &prop : props) {
std::string key, value;
int status = nntrainer::getKeyValue(prop, key, value);
if (nntrainer::istrequal(key, "engine")) {
constexpr const auto data =
std::data(props::ComputeEngineTypeInfo::EnumList);
for (uint i = 0; i < props::ComputeEngineTypeInfo::EnumList.size(); ++i) {
if (nntrainer::istrequal(value.c_str(),
props::ComputeEngineTypeInfo::EnumStr[i])) {
return data[i];
}
}
}
}

return ml::train::LayerComputeEngine::CPU;
}

/**
* @brief Layer factory creator with constructor
*/
std::unique_ptr<LayerNode>
createLayerNode(const ml::train::LayerType &type,
const std::vector<std::string> &properties,
const ml::train::LayerComputeEngine &compute_engine) {
const std::vector<std::string> &properties) {

if (getComputeEngine(properties) == ml::train::LayerComputeEngine::GPU) {
#ifdef ENABLE_OPENCL
if (compute_engine == ml::train::LayerComputeEngine::GPU) {
auto &cc = nntrainer::ClContext::Global();
return createLayerNode(cc.createObject<nntrainer::Layer>(type), properties,
compute_engine);
}
return createLayerNode(cc.createObject<nntrainer::Layer>(type), properties);
#else
throw std::invalid_argument(
"opencl layer creation without enable-opencl option");
#endif
}

auto &ac = nntrainer::AppContext::Global();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think it holds for both ml::train::LayerComputeEngine::CPU and ml::train::LayerComputeEngine::QNN?
If not, you may need to add more condition to check it.
I think it would be better to add a case to check QNN with NYI exception.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No. it doesn't. This will changed in PR #2848 too.

nntrainer/nntrainer/layers/layer_node.cpp

Line 140 in dbbde82

auto &eg = nntrainer::Engine::Global();

return createLayerNode(ac.createObject<nntrainer::Layer>(type), properties);
}
Expand All @@ -153,15 +182,18 @@ createLayerNode(const ml::train::LayerType &type,
*/
std::unique_ptr<LayerNode>
createLayerNode(const std::string &type,
const std::vector<std::string> &properties,
const ml::train::LayerComputeEngine &compute_engine) {
const std::vector<std::string> &properties) {

if (getComputeEngine(properties) == ml::train::LayerComputeEngine::GPU) {
#ifdef ENABLE_OPENCL
if (compute_engine == ml::train::LayerComputeEngine::GPU) {
auto &cc = nntrainer::ClContext::Global();
return createLayerNode(cc.createObject<nntrainer::Layer>(type), properties,
compute_engine);
}
return createLayerNode(cc.createObject<nntrainer::Layer>(type), properties);
#else
throw std::invalid_argument(
"opencl layer creation without enable-opencl option");
#endif
}

auto &ac = nntrainer::AppContext::Global();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto.

return createLayerNode(ac.createObject<nntrainer::Layer>(type), properties);
}
Expand All @@ -171,16 +203,11 @@ createLayerNode(const std::string &type,
*/
std::unique_ptr<LayerNode>
createLayerNode(std::unique_ptr<nntrainer::Layer> &&layer,
const std::vector<std::string> &properties,
const ml::train::LayerComputeEngine &compute_engine) {
const std::vector<std::string> &properties) {
auto lnode = std::make_unique<LayerNode>(std::move(layer));

lnode->setProperty(properties);

if (compute_engine == ml::train::LayerComputeEngine::GPU) {
lnode->setComputeEngine(compute_engine);
}

return lnode;
}

Expand All @@ -192,10 +219,10 @@ LayerNode::LayerNode(std::unique_ptr<nntrainer::Layer> &&l) :

output_connections(),
run_context(nullptr),
layer_node_props(
new PropsType(props::Name(), props::Distribute(), props::Trainable(), {},
{}, props::SharedFrom(), props::ClipGradByGlobalNorm(),
props::Packed(), props::LossScaleForMixed())),
layer_node_props(new PropsType(
props::Name(), props::Distribute(), props::Trainable(), {}, {},
props::SharedFrom(), props::ClipGradByGlobalNorm(), props::Packed(),
props::LossScaleForMixed(), props::ComputeEngine())),
layer_node_props_realization(
new RealizationPropsType(props::Flatten(), props::Activation())),
loss(new props::Loss()),
Expand Down Expand Up @@ -670,6 +697,10 @@ InitLayerContext LayerNode::finalize(const std::vector<TensorDim> &input_dims,
if (!std::get<props::LossScaleForMixed>(*layer_node_props).empty())
loss_scale = std::get<props::LossScaleForMixed>(*layer_node_props).get();

if (!std::get<props::ComputeEngine>(*layer_node_props).empty()) {
compute_engine = std::get<props::ComputeEngine>(*layer_node_props).get();
}

if (!std::get<props::Packed>(*layer_node_props).empty()) {
bool isPacked = std::get<props::Packed>(*layer_node_props);
if (!isPacked) {
Expand Down
24 changes: 10 additions & 14 deletions nntrainer/layers/layer_node.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ class InputConnection;
class ClipGradByGlobalNorm;
class Packed;
class LossScaleForMixed;
class ComputeEngine;
} // namespace props

/**
Expand Down Expand Up @@ -994,11 +995,12 @@ will also contain the properties of the layer. The properties will be copied
upon final creation. Editing properties of the layer after init will not the
properties in the context/graph unless intended. */

using PropsType = std::tuple<props::Name, props::Distribute, props::Trainable,
std::vector<props::InputConnection>,
std::vector<props::InputShape>,
props::SharedFrom, props::ClipGradByGlobalNorm,
props::Packed, props::LossScaleForMixed>;
using PropsType =
std::tuple<props::Name, props::Distribute, props::Trainable,
std::vector<props::InputConnection>,
std::vector<props::InputShape>, props::SharedFrom,
props::ClipGradByGlobalNorm, props::Packed,
props::LossScaleForMixed, props::ComputeEngine>;

using RealizationPropsType = std::tuple<props::Flatten, props::Activation>;
/** these realization properties results in addition of new layers, hence
Expand Down Expand Up @@ -1070,9 +1072,7 @@ properties in the context/graph unless intended. */
*/
std::unique_ptr<LayerNode>
createLayerNode(const ml::train::LayerType &type,
const std::vector<std::string> &properties = {},
const ml::train::LayerComputeEngine &compute_engine =
ml::train::LayerComputeEngine::CPU);
const std::vector<std::string> &properties = {});

/**
* @brief LayerNode creator with constructor
Expand All @@ -1082,9 +1082,7 @@ createLayerNode(const ml::train::LayerType &type,
*/
std::unique_ptr<LayerNode>
createLayerNode(const std::string &type,
const std::vector<std::string> &properties = {},
const ml::train::LayerComputeEngine &compute_engine =
ml::train::LayerComputeEngine::CPU);
const std::vector<std::string> &properties = {});

/**
* @brief LayerNode creator with constructor
Expand All @@ -1095,9 +1093,7 @@ createLayerNode(const std::string &type,
*/
std::unique_ptr<LayerNode>
createLayerNode(std::unique_ptr<nntrainer::Layer> &&layer,
const std::vector<std::string> &properties,
const ml::train::LayerComputeEngine &compute_engine =
ml::train::LayerComputeEngine::CPU);
const std::vector<std::string> &properties);

} // namespace nntrainer
#endif // __LAYER_NODE_H__
2 changes: 1 addition & 1 deletion nntrainer/utils/node_exporter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ void Exporter::saveTflResult(
std::vector<props::InputConnection>,
std::vector<props::InputShape>, props::SharedFrom,
props::ClipGradByGlobalNorm, props::Packed,
props::LossScaleForMixed> &props,
props::LossScaleForMixed, props::ComputeEngine> &props,
const LayerNode *self) {
createIfNull(tf_node);
tf_node->setLayerNode(*self);
Expand Down
Loading
Loading