diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..2abb4db --- /dev/null +++ b/.flake8 @@ -0,0 +1,22 @@ +[flake8] +max-line-length = 120 + +extend-ignore = + C101, # Coding magic comment + D100, # Missing docstring in public module + D104, # Missing docstring in public package + D401, # First line should be in imperative mood + +per-file-ignores = + sample_code.py: D100, D101, D102, D103, D104 + sample_code_with_logs.py: D100, D101, D102, D103, D104 + +exclude = + ./.git, + ./venv, + ./_data, + ./dist, + ./unit_tests, + +max-complexity = 10 +optional-ascii-coding = True diff --git a/README.md b/README.md index 6d731fc..919f2b9 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ pip install analogvnn ![3 Layered Linear Photonic Analog Neural Network](docs/_static/analogvnn_model.png) +[//]: # (![3 Layered Linear Photonic Analog Neural Network](https://github.com/Vivswan/AnalogVNN/raw/v1.0.0/docs/_static/analogvnn_model.png)) + ## Abstract **AnalogVNN** is a simulation framework built on PyTorch which can simulate the effects of @@ -53,5 +55,5 @@ Or in textual form: ```text Vivswan Shah, and Nathan Youngblood. "AnalogVNN: A fully modular framework for modeling -and optimizing photonic neural networks." *arXiv preprint arXiv:2210.10048 (2022)*. +and optimizing photonic neural networks." arXiv preprint arXiv:2210.10048 (2022). ``` \ No newline at end of file diff --git a/analogvnn/__init__.py b/analogvnn/__init__.py index 32db9f6..73955d4 100644 --- a/analogvnn/__init__.py +++ b/analogvnn/__init__.py @@ -2,13 +2,14 @@ import importlib_metadata -__package__ = "analogvnn" +__package__ = 'analogvnn' __version__ = importlib_metadata.version(__package__) -__author__ = "Vivswan Shah (vivswanshah@pitt.edu)" +__author__ = 'Vivswan Shah (vivswanshah@pitt.edu)' if sys.version_info < (3, 7, 0): import warnings warnings.warn( - 'The installed Python version reached its end-of-life. Please upgrade to a newer Python version for receiving ' + 'The installed Python version reached its end-of-life. ' + 'Please upgrade to a newer Python version for receiving ' 'further gdshelpers updates.', Warning) diff --git a/analogvnn/backward/BackwardFunction.py b/analogvnn/backward/BackwardFunction.py index be1e185..0760003 100644 --- a/analogvnn/backward/BackwardFunction.py +++ b/analogvnn/backward/BackwardFunction.py @@ -17,6 +17,7 @@ class BackwardFunction(BackwardModule, ABC): Attributes: _backward_function (Callable): The function used to compute the backward gradient. """ + _backward_function: Callable def __init__(self, backward_function: Callable, layer: nn.Module = None): @@ -26,7 +27,7 @@ def __init__(self, backward_function: Callable, layer: nn.Module = None): backward_function (Callable): The function used to compute the backward gradient. layer (nn.Module): The layer that this backward module is associated with. """ - super().__init__(layer) + super(BackwardFunction, self).__init__(layer) self._backward_function = backward_function @property @@ -60,8 +61,7 @@ def set_backward_function(self, backward_function: Callable) -> BackwardFunction return self def backward(self, *grad_output: Tensor, **grad_output_kwarg: Tensor) -> TENSORS: - """Computes the backward gradient of the input of the layer with respect to the output of the layer - using the backward function. + """Computes the backward gradient of inputs with respect to outputs using the backward function. Args: *grad_output (Tensor): The gradients of the output of the layer. @@ -74,6 +74,6 @@ def backward(self, *grad_output: Tensor, **grad_output_kwarg: Tensor) -> TENSORS NotImplementedError: If the backward function is not set. """ if self._backward_function is None: - raise ValueError("set backward_function first before invoking backward") + raise ValueError('set backward_function first before invoking backward') return self._backward_function(*grad_output, **grad_output_kwarg) diff --git a/analogvnn/backward/BackwardIdentity.py b/analogvnn/backward/BackwardIdentity.py index dc7101c..f8c9e61 100644 --- a/analogvnn/backward/BackwardIdentity.py +++ b/analogvnn/backward/BackwardIdentity.py @@ -9,8 +9,7 @@ class BackwardIdentity(BackwardModule, ABC): - """The backward module that returns the output gradients as the input gradients. - """ + """The backward module that returns the output gradients as the input gradients.""" def backward(self, *grad_output: Tensor, **grad_output_kwarg: Tensor) -> TENSORS: """Returns the output gradients as the input gradients. diff --git a/analogvnn/backward/BackwardModule.py b/analogvnn/backward/BackwardModule.py index 7789ee0..ed1fccf 100644 --- a/analogvnn/backward/BackwardModule.py +++ b/analogvnn/backward/BackwardModule.py @@ -25,16 +25,15 @@ class BackwardModule(abc.ABC): _autograd_backward (Type[AutogradBackward]): The autograd backward function. _disable_autograd_backward (bool): If True the autograd backward function is disabled. """ + _layer: Optional[nn.Module] - _empty_holder_tensor: Tensor - _autograd_backward: Type[AutogradBackward] - _disable_autograd_backward: bool + _empty_holder_tensor: Tensor = torch.ones((1,), requires_grad=True) + _autograd_backward: Type[AutogradBackward] = None + _disable_autograd_backward: bool = False # noinspection PyAbstractClass class AutogradBackward(autograd.Function): - """Autograd function is used as an optimization and proper calculation of gradients - when using the autograd engine. - """ + """Optimization and proper calculation of gradients when using the autograd engine.""" # noinspection PyMethodOverriding @staticmethod @@ -65,7 +64,6 @@ def backward(ctx: Any, *grad_outputs: Tensor) -> Tuple[None, None, TENSORS]: Returns: TENSORS: The gradients of the input of the function. """ - backward_module: BackwardModule = ctx.backward_module results = backward_module._call_impl_backward(*grad_outputs) @@ -82,12 +80,7 @@ def __init__(self, layer: nn.Module = None): """ super(BackwardModule, self).__init__() self._layer = None - self._empty_holder_tensor = torch.ones((1,), requires_grad=True) - self._empty_holder_tensor.names = ('_empty_holder_tensor',) - # noinspection PyTypeChecker - self._autograd_backward = None - self._autograd_backward = self._set_autograd_backward() - self._disable_autograd_backward = False + self._set_autograd_backward() if not isinstance(self, nn.Module): self.set_layer(layer) @@ -104,7 +97,7 @@ def forward(self, *inputs: Tensor, **inputs_kwarg: Tensor) -> TENSORS: Raises: NotImplementedError: If the forward pass is not implemented. """ - raise NotImplementedError(f"Module [{type(self).__name__}] is missing the required \"forward\" function") + raise NotImplementedError(f'Module [{type(self).__name__}] is missing the required "forward" function') forward._implemented = False @@ -122,7 +115,7 @@ def backward(self, *grad_outputs: Tensor, **grad_output_kwarg: Tensor) -> TENSOR Raises: NotImplementedError: If the backward pass is not implemented. """ - raise NotImplementedError(f"Module [{type(self).__name__}] is missing the required \"backward\" function") + raise NotImplementedError(f'Module [{type(self).__name__}] is missing the required "backward" function') def _call_impl_forward(self, *args: Tensor, **kwarg: Tensor) -> TENSORS: """Calls Forward pass of the layer. @@ -173,7 +166,7 @@ def has_forward(self) -> bool: Returns: bool: True if the forward pass is implemented, False otherwise. """ - return not hasattr(self.forward, "_implemented") + return not hasattr(self.forward, '_implemented') @property def layer(self) -> Optional[nn.Module]: @@ -210,11 +203,11 @@ def set_layer(self, layer: Optional[nn.Module]) -> BackwardModule: ValueError: If the layer is not an instance of nn.Module. """ if isinstance(self, nn.Module): - raise ValueError(f"layer of Backward Module is set to itself") + raise ValueError('layer of Backward Module is set to itself') if self._layer is not None: - raise ValueError(f"changing the layer of Backward Module is not allowed") + raise ValueError('changing the layer of Backward Module is not allowed') if layer is not None and not isinstance(layer, nn.Module): - raise ValueError(f'layer not instance of Layer class') + raise ValueError('layer not instance of Layer class') self._layer = layer self._set_autograd_backward() @@ -227,7 +220,7 @@ def _set_autograd_backward(self): else: # noinspection PyTypeChecker self._autograd_backward = type( - f"{layer.__class__.__name__}AutoGrad", + f'{layer.__class__.__name__}AutoGrad', (BackwardModule.AutogradBackward,), {} ) @@ -252,7 +245,7 @@ def set_grad_of(tensor: Tensor, grad: Tensor) -> Optional[Tensor]: tensor.backward(gradient=grad, inputs=tensor) except Exception: # noinspection PyProtectedMember - for key, value in tensor._backward_hooks.items(): + for _, value in tensor._backward_hooks.items(): grad = value(grad) if tensor.grad is None: @@ -276,6 +269,6 @@ def __getattr__(self, name: str) -> Any: """ if isinstance(self, nn.Module) or self == self._layer: return super(BackwardModule, self).__getattr__(name) - if not str(name).startswith("__") and self._layer is not None and hasattr(self._layer, name): + if not str(name).startswith('__') and self._layer is not None and hasattr(self._layer, name): return getattr(self._layer, name) raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, name)) diff --git a/analogvnn/backward/BackwardUsingForward.py b/analogvnn/backward/BackwardUsingForward.py index 79c3339..819eff7 100644 --- a/analogvnn/backward/BackwardUsingForward.py +++ b/analogvnn/backward/BackwardUsingForward.py @@ -9,12 +9,10 @@ class BackwardUsingForward(BackwardModule, ABC): - """The backward module that uses the forward function to compute the backward gradient. - """ + """The backward module that uses the forward function to compute the backward gradient.""" def backward(self, *grad_output: Tensor, **grad_output_kwarg: Tensor) -> TENSORS: - """Computes the backward gradient of the input of the layer with respect to the output of the layer - using the forward function. + """Computes the backward gradient of inputs with respect to outputs using the forward function. Args: *grad_output (Tensor): The gradients of the output of the layer. diff --git a/analogvnn/fn/reduce_precision.py b/analogvnn/fn/reduce_precision.py index 120416e..2d7ccc3 100644 --- a/analogvnn/fn/reduce_precision.py +++ b/analogvnn/fn/reduce_precision.py @@ -7,14 +7,13 @@ def reduce_precision(x: TENSOR_OPERABLE, precision: TENSOR_OPERABLE, divide: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """`reduce_precision` takes `x` and reduces its precision to `precision` by rounding to the - nearest multiple of `precision`. - + """Takes `x` and reduces its precision to `precision` by rounding to the nearest multiple of `precision`. + Args: x (TENSOR_OPERABLE): Tensor precision (TENSOR_OPERABLE): the precision of the quantization. divide (TENSOR_OPERABLE): the number of bits to be reduced - + Returns: TENSOR_OPERABLE: TENSOR_OPERABLE with the same shape as x, but with values rounded to the nearest multiple of precision. @@ -29,9 +28,8 @@ def reduce_precision(x: TENSOR_OPERABLE, precision: TENSOR_OPERABLE, divide: TEN def stochastic_reduce_precision(x: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """`stochastic_reduce_precision` takes `x` and reduces its precision to `precision` by rounding to the - nearest multiple of `precision` with a stochastic rounding scheme. - + """Takes `x` and reduces its precision by rounding to the nearest multiple of `precision` with stochastic scheme. + Args: x (TENSOR_OPERABLE): Tensor precision (TENSOR_OPERABLE): the precision of the quantization. diff --git a/analogvnn/fn/test.py b/analogvnn/fn/test.py index a2087a3..c4496cf 100644 --- a/analogvnn/fn/test.py +++ b/analogvnn/fn/test.py @@ -18,7 +18,6 @@ def test(model: nn.Module, test_loader: DataLoader, test_run: bool = False) -> T Returns: tuple: the loss and accuracy of the model on the test set. """ - model.eval() total_loss = 0 total_accuracy = 0 diff --git a/analogvnn/graph/AccumulateGrad.py b/analogvnn/graph/AccumulateGrad.py index 8d91cc1..d46b9c9 100644 --- a/analogvnn/graph/AccumulateGrad.py +++ b/analogvnn/graph/AccumulateGrad.py @@ -10,14 +10,16 @@ class AccumulateGrad: - """AccumulateGrad is a module that accumulates the gradients of the outputs of the module - it is attached to. It has no parameters of its own. + """AccumulateGrad is a module that accumulates the gradients of the outputs of the module it is attached to. + + It has no parameters of its own. Attributes: module (nn.Module): Module to accumulate gradients for. input_output_connections (Dict[str, Dict[str, Union[None, bool, int, str, GRAPH_NODE_TYPE]]]): input/output connections. """ + input_output_connections: Dict[str, Dict[str, Union[None, bool, int, str, GRAPH_NODE_TYPE]]] module: Union[nn.Module, Callable] @@ -27,6 +29,7 @@ def __init__(self, module: Union[nn.Module, Callable]): Args: module (Union[nn.Module, Callable]): Module from which to accumulate gradients. """ + super(AccumulateGrad, self).__init__() self.input_output_connections = {} self.module = module @@ -36,9 +39,9 @@ def __repr__(self): Returns: str: String representation of the module. """ - return f"AccumulateGrad({self.module})" + return f'AccumulateGrad({self.module})' - def __call__( + def __call__( # noqa: C901 self, grad_outputs_args_kwargs: ArgsKwargs, forward_input_output_graph: Dict[GRAPH_NODE_TYPE, InputOutput] @@ -128,10 +131,10 @@ def __call__( grad_inputs_args[forward_in_arg] += grad_output continue - raise NotImplementedError("WTF!Why!") + raise NotImplementedError('WTF!Why!') return ArgsKwargs( - args=[grad_inputs_args[i] for i in sorted(list(grad_inputs_args.keys()))], + args=[grad_inputs_args[i] for i in sorted(grad_inputs_args.keys())], kwargs=grad_inputs_kwargs ) diff --git a/analogvnn/graph/AcyclicDirectedGraph.py b/analogvnn/graph/AcyclicDirectedGraph.py index c6c87a1..3477a8d 100644 --- a/analogvnn/graph/AcyclicDirectedGraph.py +++ b/analogvnn/graph/AcyclicDirectedGraph.py @@ -30,6 +30,7 @@ class AcyclicDirectedGraph(abc.ABC): STOP (GraphEnum): GraphEnum.STOP """ + graph: nx.MultiDiGraph graph_state: ModelGraphState _is_static: bool @@ -48,17 +49,18 @@ def __init__(self, graph_state: ModelGraphState = None): Raises: NotImplementedError: If allow_loops is True, since this is not implemented yet. """ + super(AcyclicDirectedGraph, self).__init__() self.graph = nx.MultiDiGraph() self.graph_state = graph_state self._is_static = False self._static_graphs = {} if self.graph_state.allow_loops: - raise NotImplementedError("Loops are not implemented yet. Coming soon...") + raise NotImplementedError('Loops are not implemented yet. Coming soon...') @abc.abstractmethod def __call__(self, *args, **kwargs): - """Performs pass through the graph + """Performs pass through the graph. Args: *args: Arguments @@ -111,9 +113,9 @@ def add_edge( to_remove = [] for key, edge in existing_edges.items(): if not ( - edge["out_arg"] == attr["out_arg"] is not None + edge['out_arg'] == attr['out_arg'] is not None or - edge["out_kwarg"] == attr["out_kwarg"] is not None + edge['out_kwarg'] == attr['out_kwarg'] is not None ): continue to_remove.append(key) @@ -121,11 +123,11 @@ def add_edge( self.graph.remove_edge(u_of_edge, v_of_edge, key=key) self.graph.add_edge(u_of_edge, v_of_edge, **attr) - self.graph.nodes[u_of_edge]["fillcolor"] = "lightblue" - self.graph.nodes[v_of_edge]["fillcolor"] = "lightblue" + self.graph.nodes[u_of_edge]['fillcolor'] = 'lightblue' + self.graph.nodes[v_of_edge]['fillcolor'] = 'lightblue' return self - @staticmethod + @staticmethod # noqa: C901 def check_edge_parameters( in_arg: Union[None, int, bool], in_kwarg: Union[None, str, bool], @@ -170,16 +172,16 @@ def check_edge_parameters( if in_arg is True or in_kwarg is True: # All -> All if in_arg not in [True, None]: - raise ValueError(f'Invalid value for in_arg: "{in_arg}') + raise ValueError(f'Invalid value for in_arg: "{in_arg}"') if in_kwarg not in [True, None]: - raise ValueError(f'Invalid value for in_kwarg: "{in_kwarg}') + raise ValueError(f'Invalid value for in_kwarg: "{in_kwarg}"') if out_arg not in [True, None]: - raise ValueError(f'Invalid value for out_arg: "{out_arg}') + raise ValueError(f'Invalid value for out_arg: "{out_arg}"') if out_kwarg not in [True, None]: - raise ValueError(f'Invalid value for out_kwarg: "{out_kwarg}') + raise ValueError(f'Invalid value for out_kwarg: "{out_kwarg}"') if in_arg is True and out_kwarg is True: - raise ValueError(f'Invalid value in_arg -> out_kwarg') + raise ValueError('Invalid value "in_arg -> out_kwarg" not allowed') if (in_arg is True or in_kwarg is True) and (out_arg is None and out_kwarg is None): out_arg = in_arg @@ -206,12 +208,12 @@ def check_edge_parameters( out_kwarg = True attr = { - "in_arg": in_arg, - "in_kwarg": in_kwarg, - "out_arg": out_arg, - "out_kwarg": out_kwarg, + 'in_arg': in_arg, + 'in_kwarg': in_kwarg, + 'out_arg': out_arg, + 'out_kwarg': out_kwarg, } - attr["label"] = AcyclicDirectedGraph._create_edge_label(**attr) + attr['label'] = AcyclicDirectedGraph._create_edge_label(**attr) return attr @@ -235,28 +237,28 @@ def _create_edge_label( Returns: str: The edge's label. """ - label = "" + label = '' if in_arg == in_kwarg == out_arg == out_kwarg is True: - return "* -> *" + return '* -> *' if in_arg is True: - label += "[]" + label += '[]' elif in_arg is not None: - label += "[" + str(in_arg) + "]" + label += '[' + str(in_arg) + ']' if in_kwarg is True: - label += "{}" + label += '{}' elif in_kwarg is not None: - label += "{" + str(in_kwarg) + "}" + label += '{' + str(in_kwarg) + '}' - label += " -> " + label += ' -> ' if out_arg is True: - label += "[]" + label += '[]' elif out_arg is not None: - label += "[" + str(out_arg) + "]" + label += '[' + str(out_arg) + ']' if out_kwarg is True: - label += "{}" + label += '{}' elif out_kwarg is not None: - label += "{" + str(out_kwarg) + "}" + label += '{' + str(out_kwarg) + '}' return label @@ -297,16 +299,16 @@ def _reindex_out_args(graph: nx.MultiDiGraph) -> nx.MultiDiGraph: args_index = [] for u in graph.predecessors(v): for _, edge_data in graph.get_edge_data(u, v).items(): - if isinstance(edge_data["out_arg"], int) and not isinstance(edge_data["out_arg"], bool): + if isinstance(edge_data['out_arg'], int) and not isinstance(edge_data['out_arg'], bool): args_index.append(edge_data) if len(args_index) == 0: continue - args_index = sorted(args_index, key=lambda x: x["out_arg"]) + args_index = sorted(args_index, key=lambda x: x['out_arg']) for index, value in enumerate(args_index): - value["out_arg"] = index - value["label"] = AcyclicDirectedGraph._create_edge_label(**value) + value['out_arg'] = index + value['label'] = AcyclicDirectedGraph._create_edge_label(**value) return graph @@ -340,7 +342,7 @@ def _create_static_sub_graph( return self._static_graphs[from_node] - def parse_args_kwargs( + def parse_args_kwargs( # noqa: C901 self, input_output_graph: Dict[GRAPH_NODE_TYPE, InputOutput], module: GRAPH_NODE_TYPE, @@ -361,12 +363,12 @@ def parse_args_kwargs( kwargs = {} for p in predecessors: edge_data = self.graph.get_edge_data(p, module) - for k, v in edge_data.items(): + for _, v in edge_data.items(): previous_outputs = input_output_graph[p].outputs - in_arg = v["in_arg"] - in_kwarg = v["in_kwarg"] - out_arg = v["out_arg"] - out_kwarg = v["out_kwarg"] + in_arg = v['in_arg'] + in_kwarg = v['in_kwarg'] + out_arg = v['out_arg'] + out_kwarg = v['out_kwarg'] # 0 if in_arg is True and out_arg is True and in_kwarg is True and out_kwarg is True: @@ -414,7 +416,7 @@ def parse_args_kwargs( kwargs[out_kwarg] = previous_outputs continue - raise NotImplementedError("WTF!Why!") + raise NotImplementedError('WTF!Why!') args = [args[k] for k in sorted(args.keys())] + extra_args inputs = ArgsKwargs( diff --git a/analogvnn/graph/ArgsKwargs.py b/analogvnn/graph/ArgsKwargs.py index 9014fe7..61c1116 100644 --- a/analogvnn/graph/ArgsKwargs.py +++ b/analogvnn/graph/ArgsKwargs.py @@ -14,6 +14,7 @@ class InputOutput: inputs (Optional[ArgsKwargs]): Inputs of a module. outputs (Optional[ArgsKwargs]): Outputs of a module. """ + inputs: Optional[ArgsKwargs] = None outputs: Optional[ArgsKwargs] = None @@ -26,6 +27,7 @@ class ArgsKwargs: args (List): The arguments. kwargs (Dict): The keyword arguments. """ + args: List kwargs: Dict @@ -36,6 +38,7 @@ def __init__(self, args=None, kwargs=None): args: The arguments. kwargs: The keyword arguments. """ + super(ArgsKwargs, self).__init__() if args is None: args = [] if kwargs is None: @@ -56,11 +59,11 @@ def is_empty(self): def __repr__(self): """Returns a string representation of the parameter.""" - return f"ArgsKwargs(args={self.args}, kwargs={self.kwargs})" + return f'ArgsKwargs(args={self.args}, kwargs={self.kwargs})' @classmethod def to_args_kwargs_object(cls, outputs: ArgsKwargsInput) -> ArgsKwargs: - """ Convert the output of a module to ArgsKwargs object + """Convert the output of a module to ArgsKwargs object. Args: outputs: The output of a module @@ -80,7 +83,7 @@ def to_args_kwargs_object(cls, outputs: ArgsKwargsInput) -> ArgsKwargs: @staticmethod def from_args_kwargs_object(outputs: ArgsKwargs) -> ArgsKwargsOutput: - """Convert ArgsKwargs to object or tuple or dict + """Convert ArgsKwargs to object or tuple or dict. Args: outputs (ArgsKwargs): ArgsKwargs object diff --git a/analogvnn/graph/BackwardGraph.py b/analogvnn/graph/BackwardGraph.py index 2605b81..7aff94c 100644 --- a/analogvnn/graph/BackwardGraph.py +++ b/analogvnn/graph/BackwardGraph.py @@ -22,7 +22,7 @@ class BackwardGraph(AcyclicDirectedGraph): """The backward graph.""" def __call__(self, gradient: TENSORS = None) -> ArgsKwargsOutput: - """Backward pass through the backward graph + """Backward pass through the backward graph. Args: gradient (TENSORS): gradient of the loss function w.r.t. the output of the forward graph @@ -53,7 +53,7 @@ def __call__(self, gradient: TENSORS = None) -> ArgsKwargsOutput: return result def compile(self, is_static=True): - """Compile the graph + """Compile the graph. Args: is_static (bool): If True, the graph is not changing during runtime and will be cached. @@ -69,8 +69,8 @@ def compile(self, is_static=True): return super().compile(is_static=is_static) - def from_forward(self, forward_graph: Union[AcyclicDirectedGraph, nx.DiGraph]) -> BackwardGraph: - """Create a backward graph from inverting forward graph + def from_forward(self, forward_graph: Union[AcyclicDirectedGraph, nx.DiGraph]) -> BackwardGraph: # noqa: C901 + """Create a backward graph from inverting forward graph. Args: forward_graph (Union[AcyclicDirectedGraph, nx.DiGraph]): The forward graph. @@ -83,9 +83,9 @@ def from_forward(self, forward_graph: Union[AcyclicDirectedGraph, nx.DiGraph]) - graph = forward_graph.reverse(copy=True) for _, _, attr in graph.edges(data=True): - attr["in_arg"], attr["out_arg"] = attr["out_arg"], attr["in_arg"] - attr["in_kwarg"], attr["out_kwarg"] = attr["out_kwarg"], attr["in_kwarg"] - attr["label"] = AcyclicDirectedGraph._create_edge_label(**attr) + attr['in_arg'], attr['out_arg'] = attr['out_arg'], attr['in_arg'] + attr['in_kwarg'], attr['out_kwarg'] = attr['out_kwarg'], attr['in_kwarg'] + attr['label'] = AcyclicDirectedGraph._create_edge_label(**attr) new_graph = nx.MultiDiGraph() for v in graph.nodes(): @@ -95,75 +95,75 @@ def from_forward(self, forward_graph: Union[AcyclicDirectedGraph, nx.DiGraph]) - if len(all_predecessors) == 1 and len(graph.get_edge_data(all_predecessors[0], v)) == 1: attr = graph.get_edge_data(all_predecessors[0], v)[0] - if attr["in_arg"] == attr["in_kwarg"] == attr["in_arg"] == attr["in_arg"] is True: + if attr['in_arg'] == attr['in_kwarg'] == attr['in_arg'] == attr['in_arg'] is True: new_graph.add_edge(all_predecessors[0], v, **attr) continue akc = AccumulateGrad(v) for u in all_predecessors: - for key, attr in graph.get_edge_data(u, v).items(): - if attr["in_arg"] is None or attr["in_kwarg"] is None: - uuid_str = str(uuid.uuid4()).replace("-", "") + for _, attr in graph.get_edge_data(u, v).items(): + if attr['in_arg'] is None or attr['in_kwarg'] is None: + uuid_str = str(uuid.uuid4()).replace('-', '') new_graph.add_edge(u, akc, **{ - "in_arg": attr["in_arg"], - "in_kwarg": attr["in_kwarg"], - "out_arg": None, - "out_kwarg": uuid_str, - "real_label": " ".join(attr["label"].split(" ")[:-1] + ["{" + uuid_str + "}"]), - "label": attr["label"] + 'in_arg': attr['in_arg'], + 'in_kwarg': attr['in_kwarg'], + 'out_arg': None, + 'out_kwarg': uuid_str, + 'real_label': ' '.join(attr['label'].split(' ')[:-1] + ['{' + uuid_str + '}']), + 'label': attr['label'] }) akc.input_output_connections[uuid_str] = { **attr, - "from": u, + 'from': u, } else: - uuid_str = str(uuid.uuid4()).replace("-", "") + uuid_str = str(uuid.uuid4()).replace('-', '') new_graph.add_edge(u, akc, **{ - "in_arg": True, - "in_kwarg": None, - "out_arg": None, - "out_kwarg": uuid_str, - "real_label": "[] -> {" + uuid_str + "}", - "label": "[] -> []", + 'in_arg': True, + 'in_kwarg': None, + 'out_arg': None, + 'out_kwarg': uuid_str, + 'real_label': '[] -> {' + uuid_str + '}', + 'label': '[] -> []', }) akc.input_output_connections[uuid_str] = { **attr, - "in_kwarg": None, - "out_kwarg": None, - "from": u, + 'in_kwarg': None, + 'out_kwarg': None, + 'from': u, } - uuid_str = str(uuid.uuid4()).replace("-", "") + uuid_str = str(uuid.uuid4()).replace('-', '') new_graph.add_edge(u, akc, **{ - "in_arg": None, - "in_kwarg": True, - "out_arg": None, - "out_kwarg": uuid_str, - "real_label": "{} -> {" + uuid_str + "}", - "label": "{} -> {}", + 'in_arg': None, + 'in_kwarg': True, + 'out_arg': None, + 'out_kwarg': uuid_str, + 'real_label': '{} -> {' + uuid_str + '}', + 'label': '{} -> {}', }) akc.input_output_connections[uuid_str] = { **attr, - "in_arg": None, - "out_arg": None, - "from": u, + 'in_arg': None, + 'out_arg': None, + 'from': u, } new_graph.add_edge(akc, v, **{ - "in_arg": True, - "in_kwarg": True, - "out_arg": True, - "out_kwarg": True, - "len": 0, + 'in_arg': True, + 'in_kwarg': True, + 'out_arg': True, + 'out_kwarg': True, + 'len': 0, }) for v in graph.nodes(): - new_graph.nodes[v]["fillcolor"] = "lightblue" + new_graph.nodes[v]['fillcolor'] = 'lightblue' self.graph = new_graph return self @torch.no_grad() def calculate(self, *args, **kwargs) -> ArgsKwargsOutput: - """Calculate the gradient of the whole graph w.r.t. loss + """Calculate the gradient of the whole graph w.r.t. loss. Args: *args: The gradients args of outputs. @@ -175,9 +175,8 @@ def calculate(self, *args, **kwargs) -> ArgsKwargsOutput: Raises: ValueError: If no forward pass has been performed yet. """ - if self.graph_state.forward_input_output_graph is None: - raise ValueError("No forward pass has been performed yet. Please preform a forward pass first.") + raise ValueError('No forward pass has been performed yet. Please preform a forward pass first.') input_output_graph = self._pass(self.OUTPUT, *args, **kwargs) self.graph_state.forward_input_output_graph = None @@ -188,7 +187,7 @@ def calculate(self, *args, **kwargs) -> ArgsKwargsOutput: return None def _pass(self, from_node: GRAPH_NODE_TYPE, *args, **kwargs) -> Dict[GRAPH_NODE_TYPE, InputOutput]: - """Perform the backward pass through the graph + """Perform the backward pass through the graph. Args: from_node (GRAPH_NODE_TYPE): The node to start the backward pass from. @@ -222,7 +221,7 @@ def _pass(self, from_node: GRAPH_NODE_TYPE, *args, **kwargs) -> Dict[GRAPH_NODE_ return input_output_graph - def _calculate_gradients( + def _calculate_gradients( # noqa: C901 self, module: Union[AccumulateGrad, Layer, BackwardModule, Callable], grad_outputs: InputOutput diff --git a/analogvnn/graph/ForwardGraph.py b/analogvnn/graph/ForwardGraph.py index 9689736..9dea69a 100644 --- a/analogvnn/graph/ForwardGraph.py +++ b/analogvnn/graph/ForwardGraph.py @@ -17,7 +17,7 @@ class ForwardGraph(AcyclicDirectedGraph): """The forward graph.""" def __call__(self, inputs: TENSORS, is_training: bool) -> ArgsKwargsOutput: - """Forward pass through the forward graph + """Forward pass through the forward graph. Args: inputs (TENSORS): Input to the graph @@ -31,7 +31,7 @@ def __call__(self, inputs: TENSORS, is_training: bool) -> ArgsKwargsOutput: return outputs def compile(self, is_static: bool = True): - """Compile the graph + """Compile the graph. Args: is_static (bool): If True, the graph is not changing during runtime and will be cached. @@ -56,7 +56,7 @@ def calculate( is_training: bool = True, **kwargs ) -> ArgsKwargsOutput: - """Calculate the output of the graph + """Calculate the output of the graph. Args: inputs (TENSORS): Input to the graph @@ -81,7 +81,7 @@ def calculate( return ArgsKwargs.from_args_kwargs_object(outputs) def _pass(self, from_node: GraphEnum, *inputs: Tensor) -> Dict[GraphEnum, InputOutput]: - """Perform the forward pass through the graph + """Perform the forward pass through the graph. Args: from_node (GraphEnum): The node to start the forward pass from @@ -116,7 +116,7 @@ def _pass(self, from_node: GraphEnum, *inputs: Tensor) -> Dict[GraphEnum, InputO @staticmethod def _detach_tensor(tensor: torch.Tensor) -> torch.Tensor: - """Detach the tensor from the autograd graph + """Detach the tensor from the autograd graph. Args: tensor (torch.Tensor): Tensor to detach diff --git a/analogvnn/graph/GraphEnum.py b/analogvnn/graph/GraphEnum.py index 1b87278..21c1b1e 100644 --- a/analogvnn/graph/GraphEnum.py +++ b/analogvnn/graph/GraphEnum.py @@ -14,9 +14,10 @@ class GraphEnum(enum.Enum): OUTPUT (GraphEnum): GraphEnum.OUTPUT STOP (GraphEnum): GraphEnum.STOP """ - INPUT = "INPUT" - OUTPUT = "OUTPUT" - STOP = "STOP" + + INPUT = 'INPUT' + OUTPUT = 'OUTPUT' + STOP = 'STOP' GRAPH_NODE_TYPE = Union[GraphEnum, Callable] diff --git a/analogvnn/graph/ModelGraph.py b/analogvnn/graph/ModelGraph.py index da2fdd9..57d7874 100644 --- a/analogvnn/graph/ModelGraph.py +++ b/analogvnn/graph/ModelGraph.py @@ -14,6 +14,7 @@ class ModelGraph(ModelGraphState): forward_graph (ForwardGraph): store model's forward graph. backward_graph (BackwardGraph): store model's backward graph. """ + forward_graph: ForwardGraph backward_graph: BackwardGraph @@ -45,4 +46,3 @@ def compile(self, is_static: bool = True, auto_backward_graph: bool = False) -> self.backward_graph.compile(is_static=is_static) return self - diff --git a/analogvnn/graph/ModelGraphState.py b/analogvnn/graph/ModelGraphState.py index 9d5c461..e7a3790 100644 --- a/analogvnn/graph/ModelGraphState.py +++ b/analogvnn/graph/ModelGraphState.py @@ -28,6 +28,7 @@ class ModelGraphState: output (Tensor): the output of the forward pass. loss (Tensor): the loss. """ + allow_loops: bool use_autograd_graph: bool forward_input_output_graph: Optional[Dict[GRAPH_NODE_TYPE, InputOutput]] @@ -44,6 +45,7 @@ def __init__(self, use_autograd_graph: bool = False, allow_loops=False): use_autograd_graph: If True, the autograd graph is used to calculate the gradients. allow_loops: If True, the graph is allowed to contain loops. """ + super(ModelGraphState, self).__init__() self.allow_loops = allow_loops self.use_autograd_graph = use_autograd_graph @@ -77,10 +79,10 @@ def ready_for_backward(self, exception: bool = False) -> bool: """ if exception: if self.outputs is None: - raise RuntimeError("output is not set.") + raise RuntimeError('output is not set.') if self._loss is None: - raise RuntimeError("loss is not set.") + raise RuntimeError('loss is not set.') else: return not (self.outputs is None or self._loss is None) diff --git a/analogvnn/graph/to_graph_viz_digraph.py b/analogvnn/graph/to_graph_viz_digraph.py index a7468d4..96fdda7 100644 --- a/analogvnn/graph/to_graph_viz_digraph.py +++ b/analogvnn/graph/to_graph_viz_digraph.py @@ -27,56 +27,59 @@ def to_graphviz_digraph(from_graph: networkx.DiGraph, real_label: bool = False) ImportError: if graphviz (https://pygraphviz.github.io/) is not available. """ try: - import pygraphviz + # noinspection PyPackageRequirements + import pygraphviz # noqa: F401 except ImportError as e: - raise ImportError("requires pygraphviz: https://pygraphviz.github.io/") from e + raise ImportError('requires pygraphviz: https://pygraphviz.github.io/') from e try: from graphviz import Digraph except ImportError as e: - raise ImportError("requires graphviz: https://pygraphviz.github.io/") from e + raise ImportError('requires graphviz: https://pygraphviz.github.io/') from e strict = networkx.number_of_selfloops(from_graph) == 0 and not from_graph.is_multigraph() - node_attr = dict(style='filled', - shape='box', - align='left', - fontsize='10', - ranksep='0.1', - height='0.2', - fontname='monospace') + node_attr = { + 'style': 'filled', + 'shape': 'box', + 'align': 'left', + 'fontsize': '10', + 'ranksep': '0.1', + 'height': '0.2', + 'fontname': 'monospace' + } new_graph = Digraph( name=from_graph.name, strict=strict, node_attr=node_attr, - graph_attr=dict(size="12,12"), - format="svg" + graph_attr={'size': '12,12'}, + format='svg' ) # default graph attributes - new_graph.graph_attr.update(from_graph.graph.get("graph", {})) - new_graph.node_attr.update(from_graph.graph.get("node", {})) - new_graph.edge_attr.update(from_graph.graph.get("edge", {})) + new_graph.graph_attr.update(from_graph.graph.get('graph', {})) + new_graph.node_attr.update(from_graph.graph.get('node', {})) + new_graph.edge_attr.update(from_graph.graph.get('edge', {})) new_graph.graph_attr.update( - (k, v) for k, v in from_graph.graph.items() if k not in ("graph", "node", "edge") + (k, v) for k, v in from_graph.graph.items() if k not in ('graph', 'node', 'edge') ) # add nodes for n, nodedata in from_graph.nodes(data=True): attr = {k: str(v) for k, v in nodedata.items()} if isinstance(n, GraphEnum): - attr["fillcolor"] = 'lightblue' - attr["name"] = str(id(n)) - attr["label"] = str(n).strip() + attr['fillcolor'] = 'lightblue' + attr['name'] = str(id(n)) + attr['label'] = str(n).strip() new_graph.node(**attr) for u, v, edgedata in from_graph.edges(data=True): attr = {k: str(v) for k, v in edgedata.items()} - attr["tail_name"] = str(id(u)) - attr["head_name"] = str(id(v)) + attr['tail_name'] = str(id(u)) + attr['head_name'] = str(id(v)) - if real_label and "real_label" in attr: - attr["label"] = attr["real_label"] + if real_label and 'real_label' in attr: + attr['label'] = attr['real_label'] else: - attr["label"] = attr["label"].replace("->", "→") if "label" in attr else None + attr['label'] = attr['label'].replace('->', '→') if 'label' in attr else None new_graph.edge(**attr) return new_graph diff --git a/analogvnn/nn/Linear.py b/analogvnn/nn/Linear.py index 2f670dc..eceb7d3 100644 --- a/analogvnn/nn/Linear.py +++ b/analogvnn/nn/Linear.py @@ -12,8 +12,7 @@ class LinearBackpropagation(BackwardModule): - """The backpropagation module of a linear layer. - """ + """The backpropagation module of a linear layer.""" def forward(self, x: Tensor): """Forward pass of the linear layer. @@ -60,6 +59,7 @@ class Linear(Layer): weight (nn.Parameter): The weight of the layer. bias (nn.Parameter): The bias of the layer. """ + __constants__ = ['in_features', 'out_features'] in_features: int out_features: int diff --git a/analogvnn/nn/activation/Activation.py b/analogvnn/nn/activation/Activation.py index 22d2f55..902194b 100644 --- a/analogvnn/nn/activation/Activation.py +++ b/analogvnn/nn/activation/Activation.py @@ -9,12 +9,11 @@ class InitImplement: - """ Implements the initialization of parameters using the activation function. - """ + """Implements the initialisation of parameters using the activation function.""" @staticmethod def initialise(tensor: Tensor) -> Tensor: - """initialize the tensor using xavier uniform initialization. + """Initialisation of tensor using xavier uniform initialisation. Args: tensor (Tensor): the tensor to be initialized. @@ -26,7 +25,7 @@ def initialise(tensor: Tensor) -> Tensor: @staticmethod def initialise_(tensor: Tensor) -> Tensor: - """in-place initialize the tensor using xavier uniform initialization. + """In-place initialisation of tensor using xavier uniform initialisation. Args: tensor (Tensor): the tensor to be initialized. @@ -38,6 +37,4 @@ def initialise_(tensor: Tensor) -> Tensor: class Activation(Layer, BackwardModule, InitImplement, abc.ABC): - """This class is base class for all activation functions. - """ - pass + """This class is base class for all activation functions.""" diff --git a/analogvnn/nn/activation/BinaryStep.py b/analogvnn/nn/activation/BinaryStep.py index 4085ded..b8684b2 100644 --- a/analogvnn/nn/activation/BinaryStep.py +++ b/analogvnn/nn/activation/BinaryStep.py @@ -9,12 +9,11 @@ class BinaryStep(Activation): - """Implements the binary step activation function. - """ + """Implements the binary step activation function.""" @staticmethod def forward(x: Tensor) -> Tensor: - """forward pass of the binary step activation function. + """Forward pass of the binary step activation function. Args: x (Tensor): the input tensor. @@ -25,7 +24,7 @@ def forward(x: Tensor) -> Tensor: return (x >= 0).type(torch.float) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the binary step activation function. + """Backward pass of the binary step activation function. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. diff --git a/analogvnn/nn/activation/ELU.py b/analogvnn/nn/activation/ELU.py index 0bc1db0..e736a2c 100644 --- a/analogvnn/nn/activation/ELU.py +++ b/analogvnn/nn/activation/ELU.py @@ -15,12 +15,13 @@ class SELU(Activation): alpha (nn.Parameter): the alpha parameter. scale_factor (nn.Parameter): the scale factor parameter. """ + __constants__ = ['alpha', 'scale_factor'] alpha: nn.Parameter scale_factor: nn.Parameter def __init__(self, alpha: float = 1.0507, scale_factor: float = 1.): - """initialize the scaled exponential linear unit (SELU) activation function. + """Initialize the scaled exponential linear unit (SELU) activation function. Args: alpha (float): the alpha parameter. @@ -31,7 +32,7 @@ def __init__(self, alpha: float = 1.0507, scale_factor: float = 1.): self.scale_factor = nn.Parameter(torch.tensor(scale_factor), requires_grad=False) def forward(self, x: Tensor) -> Tensor: - """forward pass of the scaled exponential linear unit (SELU) activation function. + """Forward pass of the scaled exponential linear unit (SELU) activation function. Args: x (Tensor): the input tensor. @@ -45,7 +46,7 @@ def forward(self, x: Tensor) -> Tensor: ) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the scaled exponential linear unit (SELU) activation function. + """Backward pass of the scaled exponential linear unit (SELU) activation function. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. @@ -59,8 +60,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @staticmethod def initialise(tensor: Tensor) -> Tensor: - """initialize the tensor using xavier uniform initialization with gain associated - with the scaled exponential linear unit (SELU) activation function. + """Initialisation of tensor using xavier uniform, gain associated with SELU activation function. Args: tensor (Tensor): the tensor to be initialized. @@ -72,8 +72,7 @@ def initialise(tensor: Tensor) -> Tensor: @staticmethod def initialise_(tensor: Tensor) -> Tensor: - """in-place initialize the tensor using xavier uniform initialization with gain associated - with the scaled exponential linear unit (SELU) activation function. + """In-place initialisation of tensor using xavier uniform, gain associated with SELU activation function. Args: tensor (Tensor): the tensor to be initialized. @@ -93,7 +92,7 @@ class ELU(SELU): """ def __init__(self, alpha: float = 1.0507): - """initialize the exponential linear unit (ELU) activation function. + """Initialize the exponential linear unit (ELU) activation function. Args: alpha (float): the alpha parameter. diff --git a/analogvnn/nn/activation/Gaussian.py b/analogvnn/nn/activation/Gaussian.py index dd73ca4..8e82c6a 100644 --- a/analogvnn/nn/activation/Gaussian.py +++ b/analogvnn/nn/activation/Gaussian.py @@ -10,12 +10,11 @@ class Gaussian(Activation): - """Implements the Gaussian activation function. - """ + """Implements the Gaussian activation function.""" @staticmethod def forward(x: Tensor) -> Tensor: - """forward pass of the Gaussian activation function. + """Forward pass of the Gaussian activation function. Args: x (Tensor): the input tensor. @@ -26,7 +25,7 @@ def forward(x: Tensor) -> Tensor: return torch.exp(-torch.pow(x, 2)) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the Gaussian activation function. + """Backward pass of the Gaussian activation function. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. @@ -40,12 +39,11 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: class GeLU(Activation): - """Implements the Gaussian error linear unit (GeLU) activation function. - """ + """Implements the Gaussian error linear unit (GeLU) activation function.""" @staticmethod def forward(x: Tensor) -> Tensor: - """forward pass of the Gaussian error linear unit (GeLU) activation function. + """Forward pass of the Gaussian error linear unit (GeLU) activation function. Args: x (Tensor): the input tensor. @@ -56,7 +54,7 @@ def forward(x: Tensor) -> Tensor: return (1 / 2) * x * (1 + torch.erf(x / math.sqrt(2))) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the Gaussian error linear unit (GeLU) activation function. + """Backward pass of the Gaussian error linear unit (GeLU) activation function. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. diff --git a/analogvnn/nn/activation/Identity.py b/analogvnn/nn/activation/Identity.py index ceb20bf..d687226 100644 --- a/analogvnn/nn/activation/Identity.py +++ b/analogvnn/nn/activation/Identity.py @@ -13,10 +13,11 @@ class Identity(Activation): Attributes: name (str): the name of the activation function. """ + name: Optional[str] def __init__(self, name=None): - """initialize the identity activation function. + """Initialize the identity activation function. Args: name (str): the name of the activation function. @@ -25,7 +26,7 @@ def __init__(self, name=None): self.name = name def extra_repr(self) -> str: - """extra __repr__ of the identity activation function. + """Extra __repr__ of the identity activation function. Returns: str: the extra representation of the identity activation function. @@ -37,7 +38,7 @@ def extra_repr(self) -> str: @staticmethod def forward(x: Tensor) -> Tensor: - """forward pass of the identity activation function. + """Forward pass of the identity activation function. Args: x (Tensor): the input tensor. @@ -48,7 +49,7 @@ def forward(x: Tensor) -> Tensor: return x def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the identity activation function. + """Backward pass of the identity activation function. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. diff --git a/analogvnn/nn/activation/ReLU.py b/analogvnn/nn/activation/ReLU.py index 238c3ec..f00d576 100644 --- a/analogvnn/nn/activation/ReLU.py +++ b/analogvnn/nn/activation/ReLU.py @@ -16,12 +16,13 @@ class PReLU(Activation): alpha (float): the slope of the negative part of the activation function. _zero (Tensor): placeholder tensor of zero. """ + __constants__ = ['alpha', '_zero'] alpha: nn.Parameter _zero: nn.Parameter def __init__(self, alpha: float): - """initialize the parametric rectified linear unit (PReLU) activation function. + """Initialize the parametric rectified linear unit (PReLU) activation function. Args: alpha (float): the slope of the negative part of the activation function. @@ -31,7 +32,7 @@ def __init__(self, alpha: float): self._zero = nn.Parameter(torch.tensor(0), requires_grad=False) def forward(self, x: Tensor) -> Tensor: - """forward pass of the parametric rectified linear unit (PReLU) activation function. + """Forward pass of the parametric rectified linear unit (PReLU) activation function. Args: x (Tensor): the input tensor. @@ -42,7 +43,7 @@ def forward(self, x: Tensor) -> Tensor: return torch.minimum(self._zero, x) * self.alpha + torch.maximum(self._zero, x) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the parametric rectified linear unit (PReLU) activation function. + """Backward pass of the parametric rectified linear unit (PReLU) activation function. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. @@ -56,8 +57,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @staticmethod def initialise(tensor: Tensor) -> Tensor: - """initialize the tensor using kaiming uniform initialization with gain associated - with the parametric rectified linear unit (PReLU) activation function. + """Initialisation of tensor using kaiming uniform, gain associated with PReLU activation function. Args: tensor (Tensor): the tensor to be initialized. @@ -65,12 +65,11 @@ def initialise(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ - return nn.init.kaiming_uniform(tensor, a=math.sqrt(5), nonlinearity="leaky_relu") + return nn.init.kaiming_uniform(tensor, a=math.sqrt(5), nonlinearity='leaky_relu') @staticmethod def initialise_(tensor: Tensor) -> Tensor: - """in-place initialize the tensor using kaiming uniform initialization with gain associated - with the parametric rectified linear unit (PReLU) activation function. + """In-place initialisation of tensor using kaiming uniform, gain associated with PReLU activation function. Args: tensor (Tensor): the tensor to be initialized. @@ -78,24 +77,23 @@ def initialise_(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ - return nn.init.kaiming_uniform_(tensor, a=math.sqrt(5), nonlinearity="leaky_relu") + return nn.init.kaiming_uniform_(tensor, a=math.sqrt(5), nonlinearity='leaky_relu') class ReLU(PReLU): """Implements the rectified linear unit (ReLU) activation function. + Attributes: alpha (float): 0 """ def __init__(self): - """initialize the rectified linear unit (ReLU) activation function. - """ + """Initialize the rectified linear unit (ReLU) activation function.""" super(ReLU, self).__init__(alpha=0) @staticmethod def initialise(tensor: Tensor) -> Tensor: - """initialize the tensor using kaiming uniform initialization with gain associated - with the rectified linear unit (ReLU) activation function. + """Initialisation of tensor using kaiming uniform, gain associated with ReLU activation function. Args: tensor (Tensor): the tensor to be initialized. @@ -103,11 +101,19 @@ def initialise(tensor: Tensor) -> Tensor: Returns: Tensor: the initialized tensor. """ - return nn.init.kaiming_uniform(tensor, a=math.sqrt(5), nonlinearity="relu") + return nn.init.kaiming_uniform(tensor, a=math.sqrt(5), nonlinearity='relu') @staticmethod def initialise_(tensor: Tensor) -> Tensor: - return nn.init.kaiming_uniform_(tensor, a=math.sqrt(5), nonlinearity="relu") + """In-place initialisation of tensor using kaiming uniform, gain associated with ReLU activation function. + + Args: + tensor (Tensor): the tensor to be initialized. + + Returns: + Tensor: the initialized tensor. + """ + return nn.init.kaiming_uniform_(tensor, a=math.sqrt(5), nonlinearity='relu') class LeakyReLU(PReLU): @@ -118,6 +124,5 @@ class LeakyReLU(PReLU): """ def __init__(self): - """initialize the leaky rectified linear unit (LeakyReLU) activation function. - """ + """Initialize the leaky rectified linear unit (LeakyReLU) activation function.""" super(LeakyReLU, self).__init__(alpha=0.01) diff --git a/analogvnn/nn/activation/SiLU.py b/analogvnn/nn/activation/SiLU.py index 659efef..afb0265 100644 --- a/analogvnn/nn/activation/SiLU.py +++ b/analogvnn/nn/activation/SiLU.py @@ -9,12 +9,11 @@ class SiLU(Activation): - """Implements the SiLU activation function. - """ + """Implements the SiLU activation function.""" @staticmethod def forward(x: Tensor) -> Tensor: - """forward pass of the SiLU + """Forward pass of the SiLU. Args: x (Tensor): the input tensor. @@ -25,7 +24,7 @@ def forward(x: Tensor) -> Tensor: return x / (1 + torch.exp(-x)) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the SiLU + """Backward pass of the SiLU. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. diff --git a/analogvnn/nn/activation/Sigmoid.py b/analogvnn/nn/activation/Sigmoid.py index c07e622..a14e289 100644 --- a/analogvnn/nn/activation/Sigmoid.py +++ b/analogvnn/nn/activation/Sigmoid.py @@ -9,12 +9,11 @@ class Logistic(Activation): - """Implements the logistic activation function. - """ + """Implements the logistic activation function.""" @staticmethod def forward(x: Tensor) -> Tensor: - """forward pass of the logistic activation function. + """Forward pass of the logistic activation function. Args: x (Tensor): the input tensor. @@ -25,7 +24,7 @@ def forward(x: Tensor) -> Tensor: return 1 / (1 + torch.exp(-x)) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the logistic activation function. + """Backward pass of the logistic activation function. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. @@ -39,8 +38,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @staticmethod def initialise(tensor: Tensor) -> Tensor: - """initialize the tensor using xavier uniform initialization with gain associated - with the logistic activation function. + """Initialisation of tensor using xavier uniform, gain associated with logistic activation function. Args: tensor (Tensor): the tensor to be initialized. @@ -52,8 +50,7 @@ def initialise(tensor: Tensor) -> Tensor: @staticmethod def initialise_(tensor: Tensor) -> Tensor: - """in-place initialize the tensor using xavier uniform initialization with gain associated - with the logistic activation function. + """In-place initialisation of tensor using xavier uniform, gain associated with logistic activation function. Args: tensor (Tensor): the tensor to be initialized. @@ -65,6 +62,4 @@ def initialise_(tensor: Tensor) -> Tensor: class Sigmoid(Logistic): - """Implements the sigmoid activation function. - """ - pass + """Implements the sigmoid activation function.""" diff --git a/analogvnn/nn/activation/Tanh.py b/analogvnn/nn/activation/Tanh.py index 02b2286..4d7d61e 100644 --- a/analogvnn/nn/activation/Tanh.py +++ b/analogvnn/nn/activation/Tanh.py @@ -9,12 +9,11 @@ class Tanh(Activation): - """Implements the tanh activation function. - """ + """Implements the tanh activation function.""" @staticmethod def forward(x: Tensor) -> Tensor: - """forward pass of the tanh activation function. + """Forward pass of the tanh activation function. Args: x (Tensor): the input tensor. @@ -25,7 +24,7 @@ def forward(x: Tensor) -> Tensor: return torch.tanh(x) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the tanh activation function. + """Backward pass of the tanh activation function. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. @@ -39,8 +38,7 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: @staticmethod def initialise(tensor: Tensor) -> Tensor: - """initialize the tensor using xavier uniform initialization with gain associated - with the tanh activation function. + """Initialisation of tensor using xavier uniform, gain associated with tanh. Args: tensor (Tensor): the tensor to be initialized. @@ -52,8 +50,7 @@ def initialise(tensor: Tensor) -> Tensor: @staticmethod def initialise_(tensor: Tensor) -> Tensor: - """in-place initialize the tensor using xavier uniform initialization with gain associated - with the tanh activation function. + """In-place initialisation of tensor using xavier uniform, gain associated with tanh. Args: tensor (Tensor): the tensor to be initialized. diff --git a/analogvnn/nn/module/FullSequential.py b/analogvnn/nn/module/FullSequential.py index b99a7b5..1ab5a21 100644 --- a/analogvnn/nn/module/FullSequential.py +++ b/analogvnn/nn/module/FullSequential.py @@ -10,8 +10,7 @@ class FullSequential(Sequential): - """A sequential model where backward graph is the reverse of forward graph. - """ + """A sequential model where backward graph is the reverse of forward graph.""" def compile(self, device: Optional[torch.device] = None, layer_data: bool = True): """Compile the model and add forward and backward graph. diff --git a/analogvnn/nn/module/Layer.py b/analogvnn/nn/module/Layer.py index 5c5afd6..e0a07c2 100644 --- a/analogvnn/nn/module/Layer.py +++ b/analogvnn/nn/module/Layer.py @@ -18,14 +18,14 @@ # https://github.com/pytorch/pytorch/pull/91819 def __nn_Module_init_updated__(function: Callable) -> Callable: - """ Wrapper for nn.Module.__init__ to support multiple parent classes at same time. + """Wrapper for nn.Module.__init__ to support multiple parent classes at same time. + Args: function (Callable): nn.Module.__init__ function Returns: Callable: Wrapped function """ - # noinspection PyUnusedLocal def _temp(*args, **kwargs) -> ...: pass @@ -49,8 +49,9 @@ def new_function(self, *args, **kwargs): return new_function -nn.Module.__init__ = __nn_Module_init_updated__(nn.Module.__init__) -""" nn.Module.__init__ is updated to support multiple parent classes at same time. """ +if not hasattr(nn.Module, 'call_super_init'): + nn.Module.__init__ = __nn_Module_init_updated__(nn.Module.__init__) + """nn.Module.__init__ is updated to support multiple parent classes at same time. """ class Layer(nn.Module): @@ -62,13 +63,19 @@ class Layer(nn.Module): _backward_module (Optional[BackwardModule]): Backward module of the layer. _use_autograd_graph (bool): If True, the autograd graph is used to calculate the gradients. graphs (Optional[ModelGraph]): Contains Forward and Backward Graphs of the layer. + call_super_init (bool): If True, the super class __init__ of nn.Module is called + https://github.com/pytorch/pytorch/pull/91819 """ + _inputs: Union[None, ArgsKwargs] _outputs: Union[None, Tensor, Sequence[Tensor]] _backward_module: Optional[BackwardModule] _use_autograd_graph: bool graphs: Optional[ModelGraph] + # https://github.com/pytorch/pytorch/pull/91819 + call_super_init: bool = True + def __init__(self): """Initializes the layer.""" super(Layer, self).__init__() @@ -180,7 +187,7 @@ def set_backward_function(self, backward_class: Union[Callable, BackwardModule, elif callable(backward_class): self._backward_module = BackwardFunction(backward_class, self) else: - raise TypeError(f"Backward Module is not set for '{self}'") + raise TypeError(f'Backward Module is not set for "{self}"') return self @@ -194,7 +201,7 @@ def _forward_wrapper(self, function: Callable) -> Callable: Callable: Wrapped function. """ # noinspection PyUnresolvedReferences - if hasattr(function, "__wrapper__") and function.__wrapper__ == Layer._forward_wrapper: + if hasattr(function, '__wrapper__') and function.__wrapper__ == Layer._forward_wrapper: return function if not isinstance(self.backward_function, BackwardModule): @@ -213,7 +220,7 @@ def new_forward(*args, **kwargs): new_forward.__wrapped__ = function new_forward.__wrapper__ = Layer._forward_wrapper - setattr(self, "forward", new_forward) + self.forward = new_forward return new_forward def _call_impl_forward(self, *args: Tensor, **kwargs: Tensor) -> TENSORS: @@ -231,7 +238,7 @@ def _call_impl_forward(self, *args: Tensor, **kwargs: Tensor) -> TENSORS: else: forward_functions = self.forward - if hasattr(forward_functions, "__wrapped__"): + if hasattr(forward_functions, '__wrapped__'): forward_functions = forward_functions.__wrapped__ return forward_functions(*args, **kwargs) diff --git a/analogvnn/nn/module/Model.py b/analogvnn/nn/module/Model.py index c3cbe88..bb8ba0d 100644 --- a/analogvnn/nn/module/Model.py +++ b/analogvnn/nn/module/Model.py @@ -36,6 +36,7 @@ class Model(Layer): accuracy_function (Callable): The accuracy function of the model. device (torch.device): The device of the model. """ + __constants__ = ['device'] _compiled: bool @@ -77,7 +78,8 @@ def __init__(self, tensorboard_log_dir=None, device=is_cpu_cuda.device): @property def use_autograd_graph(self): - """ Is the autograd graph used for the model. + """Is the autograd graph used for the model. + Returns: bool: If True, the autograd graph is used to calculate the gradients. """ @@ -85,7 +87,7 @@ def use_autograd_graph(self): @use_autograd_graph.setter def use_autograd_graph(self, use_autograd_graph: bool): - """ Set if the autograd graph is used for the model. + """Set if the autograd graph is used for the model. Args: use_autograd_graph (bool): If True, the autograd graph is used to calculate the gradients. @@ -251,7 +253,7 @@ def create_tensorboard(self, log_dir: str) -> TensorboardModelLog: try: from analogvnn.utils.TensorboardModelLog import TensorboardModelLog except ImportError as e: - raise ImportError("requires tensorboard https://www.tensorflow.org/") from e + raise ImportError('requires tensorboard https://www.tensorflow.org/') from e self.tensorboard = TensorboardModelLog(self, log_dir=log_dir) self.subscribe_tensorboard(self.tensorboard) diff --git a/analogvnn/nn/module/Sequential.py b/analogvnn/nn/module/Sequential.py index d797d86..1ec0e67 100644 --- a/analogvnn/nn/module/Sequential.py +++ b/analogvnn/nn/module/Sequential.py @@ -24,6 +24,11 @@ class Sequential(Model): """ def __init__(self, *args): + """Initialize the model. + + Args: + *args: The modules to add. + """ super(Sequential, self).__init__() self._runtime_module_list: Dict[str, Optional[Module]] = OrderedDict() self.add_sequence(*args) @@ -67,7 +72,15 @@ def _add_run_module(self, name: str, module: Optional[Module]): return self def _get_item_by_idx(self, iterator, idx) -> T: - """Get the idx-th item of the iterator""" + """Get the idx-th item of the iterator. + + Args: + iterator (Iterator): The iterator. + idx (int): The index of the item. + + Returns: + T: The idx-th item of the iterator. + """ size = len(self) idx = operator.index(idx) if not -size <= idx < size: @@ -76,16 +89,35 @@ def _get_item_by_idx(self, iterator, idx) -> T: return next(islice(iterator, idx, None)) def __getitem__(self, idx) -> Union[Sequential, T]: + """Get the idx-th module of the model. + + Args: + idx (int): The index of the module. + + Returns: + Union[Sequential, T]: The idx-th module of the model. + """ if isinstance(idx, slice): return self.__class__(OrderedDict(list(self._runtime_module_list.items())[idx])) else: return self._get_item_by_idx(self._runtime_module_list.values(), idx) def __setitem__(self, idx: int, module: nn.Module) -> None: + """Set the idx-th module of the model. + + Args: + idx (int): The index of the module. + module (nn.Module): The module to set. + """ key: str = self._get_item_by_idx(self._runtime_module_list.keys(), idx) return setattr(self, key, module) def __delitem__(self, idx: Union[slice, int]) -> None: + """Remove the idx-th module from the model. + + Args: + idx (Union[slice, int]): The index of the module. + """ if isinstance(idx, slice): for key in list(self._runtime_module_list.keys())[idx]: delattr(self, key) @@ -94,12 +126,27 @@ def __delitem__(self, idx: Union[slice, int]) -> None: delattr(self, key) def __len__(self) -> int: + """Return the number of modules in the model. + + Returns: + int: The number of modules in the model. + """ return len(self._runtime_module_list) def __dir__(self): + """Return the list of attributes of the module. + + Returns: + list: The list of attributes of the module. + """ keys = super(Sequential, self).__dir__() keys = [key for key in keys if not key.isdigit()] return keys def __iter__(self) -> Iterator[nn.Module]: + """Return an iterator over the modules of the model. + + Returns: + Iterator[nn.Module]: An iterator over the modules of the model. + """ return iter(self._runtime_module_list.values()) diff --git a/analogvnn/nn/noise/GaussianNoise.py b/analogvnn/nn/noise/GaussianNoise.py index 46d50c6..4333332 100644 --- a/analogvnn/nn/noise/GaussianNoise.py +++ b/analogvnn/nn/noise/GaussianNoise.py @@ -16,12 +16,13 @@ class GaussianNoise(Layer, BackwardIdentity): """Implements the Gaussian noise function. - + Attributes: std (nn.Parameter): the standard deviation of the Gaussian noise. leakage (nn.Parameter): the leakage of the Gaussian noise. precision (nn.Parameter): the precision of the Gaussian noise. """ + __constants__ = ['std', 'leakage', 'precision'] std: nn.Parameter leakage: nn.Parameter @@ -33,8 +34,8 @@ def __init__( leakage: Optional[float] = None, precision: Optional[int] = None ): - """initialize the Gaussian noise function. - + """Initialize the Gaussian noise function. + Args: std (float): the standard deviation of the Gaussian noise. leakage (float): the leakage of the Gaussian noise. @@ -43,7 +44,7 @@ def __init__( super(GaussianNoise, self).__init__() if (std is None) + (leakage is None) + (precision is None) != 1: - raise ValueError("only 2 out of 3 arguments are needed (std, leakage, precision)") + raise ValueError('only 2 out of 3 arguments are needed (std, leakage, precision)') std, leakage, precision = to_float_tensor(std, leakage, precision) @@ -60,12 +61,12 @@ def __init__( @staticmethod def calc_std(leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """calculate the standard deviation of the Gaussian noise. - + """Calculate the standard deviation of the Gaussian noise. + Args: leakage (float): the leakage of the Gaussian noise. precision (int): the precision of the Gaussian noise. - + Returns: float: the standard deviation of the Gaussian noise. """ @@ -73,12 +74,12 @@ def calc_std(leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_OPE @staticmethod def calc_precision(std: TENSOR_OPERABLE, leakage: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """calculate the precision of the Gaussian noise. - + """Calculate the precision of the Gaussian noise. + Args: std (float): the standard deviation of the Gaussian noise. leakage (float): the leakage of the Gaussian noise. - + Returns: int: the precision of the Gaussian noise. """ @@ -86,12 +87,12 @@ def calc_precision(std: TENSOR_OPERABLE, leakage: TENSOR_OPERABLE) -> TENSOR_OPE @staticmethod def calc_leakage(std: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """calculate the leakage of the Gaussian noise. - + """Calculate the leakage of the Gaussian noise. + Args: std (float): the standard deviation of the Gaussian noise. precision (int): the precision of the Gaussian noise. - + Returns: float: the leakage of the Gaussian noise. """ @@ -99,8 +100,8 @@ def calc_leakage(std: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_OPE @property def stddev(self) -> Tensor: - """the standard deviation of the Gaussian noise. - + """The standard deviation of the Gaussian noise. + Returns: Tensor: the standard deviation of the Gaussian noise. """ @@ -108,32 +109,32 @@ def stddev(self) -> Tensor: @property def variance(self) -> Tensor: - """the variance of the Gaussian noise. - + """The variance of the Gaussian noise. + Returns: Tensor: the variance of the Gaussian noise. """ return self.stddev.pow(2) def pdf(self, x: Tensor, mean: Tensor = 0) -> Tensor: - """calculate the probability density function of the Gaussian noise. - + """Calculate the probability density function of the Gaussian noise. + Args: x (Tensor): the input tensor. mean (Tensor): the mean of the Gaussian noise. - + Returns: Tensor: the probability density function of the Gaussian noise. """ return torch.exp(self.log_prob(x=x, mean=mean)) def log_prob(self, x: Tensor, mean: Tensor = 0) -> Tensor: - """calculate the log probability density function of the Gaussian noise. - + """Calculate the log probability density function of the Gaussian noise. + Args: x (Tensor): the input tensor. mean (Tensor): the mean of the Gaussian noise. - + Returns: Tensor: the log probability density function of the Gaussian noise. """ @@ -146,25 +147,25 @@ def log_prob(self, x: Tensor, mean: Tensor = 0) -> Tensor: @staticmethod def static_cdf(x: TENSOR_OPERABLE, std: TENSOR_OPERABLE, mean: TENSOR_OPERABLE = 0.) -> TENSOR_OPERABLE: - """calculate the cumulative distribution function of the Gaussian noise. - + """Calculate the cumulative distribution function of the Gaussian noise. + Args: x (TENSOR_OPERABLE): the input tensor. std (TENSOR_OPERABLE): the standard deviation of the Gaussian noise. mean (TENSOR_OPERABLE): the mean of the Gaussian noise. - + Returns: TENSOR_OPERABLE: the cumulative distribution function of the Gaussian noise. """ return 1 / 2 * (1 + math.erf((x - mean) / (std * math.sqrt(2)))) def cdf(self, x: Tensor, mean: Tensor = 0) -> Tensor: - """calculate the cumulative distribution function of the Gaussian noise. - + """Calculate the cumulative distribution function of the Gaussian noise. + Args: x (Tensor): the input tensor. mean (Tensor): the mean of the Gaussian noise. - + Returns: Tensor: the cumulative distribution function of the Gaussian noise. """ @@ -173,19 +174,19 @@ def cdf(self, x: Tensor, mean: Tensor = 0) -> Tensor: return self.static_cdf(x=x, std=self.std, mean=mean) def forward(self, x: Tensor) -> Tensor: - """add the Gaussian noise to the input tensor. - + """Add the Gaussian noise to the input tensor. + Args: x (Tensor): the input tensor. - + Returns: Tensor: the output tensor. """ return torch.normal(mean=x, std=self.std) def extra_repr(self) -> str: - """the extra representation of the Gaussian noise. - + """The extra representation of the Gaussian noise. + Returns: str: the extra representation of the Gaussian noise. """ diff --git a/analogvnn/nn/noise/LaplacianNoise.py b/analogvnn/nn/noise/LaplacianNoise.py index ceda0fb..b88bc2f 100644 --- a/analogvnn/nn/noise/LaplacianNoise.py +++ b/analogvnn/nn/noise/LaplacianNoise.py @@ -21,6 +21,7 @@ class LaplacianNoise(Layer, BackwardIdentity): leakage (nn.Parameter): the leakage of the Laplacian noise. precision (nn.Parameter): the precision of the Laplacian noise. """ + __constants__ = ['scale', 'leakage', 'precision'] scale: nn.Parameter leakage: nn.Parameter @@ -32,7 +33,7 @@ def __init__( leakage: Optional[float] = None, precision: Optional[int] = None ): - """initialize the Laplacian noise function. + """Initialize the Laplacian noise function. Args: scale (float): the scale of the Laplacian noise. @@ -42,7 +43,7 @@ def __init__( super(LaplacianNoise, self).__init__() if (scale is None) + (leakage is None) + (precision is None) != 1: - raise ValueError("only 2 out of 3 arguments are needed (scale, leakage, precision)") + raise ValueError('only 2 out of 3 arguments are needed (scale, leakage, precision)') scale, leakage, precision = to_float_tensor(scale, leakage, precision) @@ -59,7 +60,7 @@ def __init__( @staticmethod def calc_scale(leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """calculate the scale of the Laplacian noise. + """Calculate the scale of the Laplacian noise. Args: leakage (float): the leakage of the Laplacian noise. @@ -72,7 +73,7 @@ def calc_scale(leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_O @staticmethod def calc_precision(scale: TENSOR_OPERABLE, leakage: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """calculate the precision of the Laplacian noise. + """Calculate the precision of the Laplacian noise. Args: scale (float): the scale of the Laplacian noise. @@ -85,7 +86,7 @@ def calc_precision(scale: TENSOR_OPERABLE, leakage: TENSOR_OPERABLE) -> TENSOR_O @staticmethod def calc_leakage(scale: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> Tensor: - """calculate the leakage of the Laplacian noise. + """Calculate the leakage of the Laplacian noise. Args: scale (float): the scale of the Laplacian noise. @@ -98,7 +99,7 @@ def calc_leakage(scale: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> Tensor: @property def stddev(self) -> Tensor: - """the standard deviation of the Laplacian noise. + """The standard deviation of the Laplacian noise. Returns: Tensor: the standard deviation of the Laplacian noise. @@ -107,7 +108,7 @@ def stddev(self) -> Tensor: @property def variance(self) -> Tensor: - """the variance of the Laplacian noise. + """The variance of the Laplacian noise. Returns: Tensor: the variance of the Laplacian noise. @@ -115,7 +116,7 @@ def variance(self) -> Tensor: return 2 * self.scale.pow(2) def pdf(self, x: TENSOR_OPERABLE, loc: TENSOR_OPERABLE = 0) -> Tensor: - """the probability density function of the Laplacian noise. + """The probability density function of the Laplacian noise. Args: x (TENSOR_OPERABLE): the input tensor. @@ -127,7 +128,7 @@ def pdf(self, x: TENSOR_OPERABLE, loc: TENSOR_OPERABLE = 0) -> Tensor: return torch.exp(self.log_prob(x=x, loc=loc)) def log_prob(self, x: TENSOR_OPERABLE, loc: TENSOR_OPERABLE = 0) -> Tensor: - """the log probability density function of the Laplacian noise. + """The log probability density function of the Laplacian noise. Args: x (TENSOR_OPERABLE): the input tensor. @@ -142,7 +143,7 @@ def log_prob(self, x: TENSOR_OPERABLE, loc: TENSOR_OPERABLE = 0) -> Tensor: @staticmethod def static_cdf(x: TENSOR_OPERABLE, scale: TENSOR_OPERABLE, loc: TENSOR_OPERABLE = 0.) -> TENSOR_OPERABLE: - """the cumulative distribution function of the Laplacian noise. + """The cumulative distribution function of the Laplacian noise. Args: x (TENSOR_OPERABLE): the input tensor. @@ -155,7 +156,7 @@ def static_cdf(x: TENSOR_OPERABLE, scale: TENSOR_OPERABLE, loc: TENSOR_OPERABLE return 0.5 - 0.5 * np.sign(x - loc) * np.expm1(-abs(x - loc) / scale) def cdf(self, x: Tensor, loc: Tensor = 0) -> Tensor: - """the cumulative distribution function of the Laplacian noise. + """The cumulative distribution function of the Laplacian noise. Args: x (Tensor): the input tensor. @@ -169,7 +170,7 @@ def cdf(self, x: Tensor, loc: Tensor = 0) -> Tensor: return self.static_cdf(x=x, scale=self.scale, loc=loc) def forward(self, x: Tensor) -> Tensor: - """add Laplacian noise to the input tensor. + """Add Laplacian noise to the input tensor. Args: x (Tensor): the input tensor. @@ -180,7 +181,7 @@ def forward(self, x: Tensor) -> Tensor: return torch.distributions.Laplace(loc=x, scale=self.scale).sample() def extra_repr(self) -> str: - """the extra representation of the Laplacian noise. + """The extra representation of the Laplacian noise. Returns: str: the extra representation of the Laplacian noise. diff --git a/analogvnn/nn/noise/PoissonNoise.py b/analogvnn/nn/noise/PoissonNoise.py index 8843823..df05371 100644 --- a/analogvnn/nn/noise/PoissonNoise.py +++ b/analogvnn/nn/noise/PoissonNoise.py @@ -3,7 +3,7 @@ import numpy as np import scipy.special import torch -from scipy.optimize import * +from scipy.optimize import toms748 from torch import Tensor, nn from analogvnn.backward.BackwardIdentity import BackwardIdentity @@ -23,6 +23,7 @@ class PoissonNoise(Layer, BackwardIdentity): max_leakage (nn.Parameter): the maximum leakage of the Poisson noise. precision (nn.Parameter): the precision of the Poisson noise. """ + __constants__ = ['scale', 'max_leakage', 'precision'] scale: nn.Parameter max_leakage: nn.Parameter @@ -44,7 +45,7 @@ def __init__( super(PoissonNoise, self).__init__() if (scale is None) + (max_leakage is None) + (precision is None) != 1: - raise ValueError("only 2 out of 3 arguments are needed (scale, max_leakage, precision)") + raise ValueError('only 2 out of 3 arguments are needed (scale, max_leakage, precision)') scale, max_leakage, precision = to_float_tensor(scale, max_leakage, precision) @@ -60,7 +61,7 @@ def __init__( self.scale, self.max_leakage, self.precision = to_nongrad_parameter(scale, max_leakage, precision) if self.rate_factor < 1.: - raise ValueError("rate_factor must be >= 1 (scale * precision >= 1)") + raise ValueError('rate_factor must be >= 1 (scale * precision >= 1)') @staticmethod def calc_scale(max_leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE, max_check=10000) -> TENSOR_OPERABLE: diff --git a/analogvnn/nn/noise/UniformNoise.py b/analogvnn/nn/noise/UniformNoise.py index e70e0f9..ac3a49d 100644 --- a/analogvnn/nn/noise/UniformNoise.py +++ b/analogvnn/nn/noise/UniformNoise.py @@ -20,6 +20,7 @@ class UniformNoise(Layer, BackwardIdentity): leakage (nn.Parameter): the leakage of the uniform noise. precision (nn.Parameter): the precision of the uniform noise. """ + __constants__ = ['low', 'high', 'leakage', 'precision'] low: nn.Parameter high: nn.Parameter @@ -33,7 +34,7 @@ def __init__( leakage: Optional[float] = None, precision: Optional[int] = None ): - """initialize the uniform noise function. + """Initialize the uniform noise function. Args: low (float): the lower bound of the uniform noise. @@ -44,7 +45,7 @@ def __init__( super(UniformNoise, self).__init__() if (low is None or high is None) + (leakage is None) + (precision is None) != 1: - raise ValueError("only 2 out of 3 arguments are needed (scale, leakage, precision)") + raise ValueError('only 2 out of 3 arguments are needed (scale, leakage, precision)') low, high, leakage, precision = to_float_tensor(low, high, leakage, precision) @@ -61,7 +62,7 @@ def __init__( @staticmethod def calc_high_low(leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> Tuple[TENSOR_OPERABLE, TENSOR_OPERABLE]: - """calculate the high and low from leakage and precision. + """Calculate the high and low from leakage and precision. Args: leakage (TENSOR_OPERABLE): the leakage of the uniform noise. @@ -75,7 +76,7 @@ def calc_high_low(leakage: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> Tuple @staticmethod def calc_leakage(low: TENSOR_OPERABLE, high: TENSOR_OPERABLE, precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """calculate the leakage from low, high and precision. + """Calculate the leakage from low, high and precision. Args: low (TENSOR_OPERABLE): the lower bound of the uniform noise. @@ -89,7 +90,7 @@ def calc_leakage(low: TENSOR_OPERABLE, high: TENSOR_OPERABLE, precision: TENSOR_ @staticmethod def calc_precision(low: TENSOR_OPERABLE, high: TENSOR_OPERABLE, leakage: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """calculate the precision from low, high and leakage. + """Calculate the precision from low, high and leakage. Args: low (TENSOR_OPERABLE): the lower bound of the uniform noise. @@ -103,7 +104,7 @@ def calc_precision(low: TENSOR_OPERABLE, high: TENSOR_OPERABLE, leakage: TENSOR_ @property def mean(self) -> Tensor: - """the mean of the uniform noise. + """The mean of the uniform noise. Returns: Tensor: the mean of the uniform noise. @@ -112,7 +113,7 @@ def mean(self) -> Tensor: @property def stddev(self) -> Tensor: - """the standard deviation of the uniform noise. + """The standard deviation of the uniform noise. Returns: Tensor: the standard deviation of the uniform noise. @@ -121,7 +122,7 @@ def stddev(self) -> Tensor: @property def variance(self) -> Tensor: - """the variance of the uniform noise. + """The variance of the uniform noise. Returns: Tensor: the variance of the uniform noise. @@ -129,7 +130,7 @@ def variance(self) -> Tensor: return (self.high - self.low).pow(2) / 12 def pdf(self, x: Tensor) -> Tensor: - """the probability density function of the uniform noise. + """The probability density function of the uniform noise. Args: x (Tensor): the input tensor. @@ -140,7 +141,7 @@ def pdf(self, x: Tensor) -> Tensor: return torch.exp(self.log_prob(x=x)) def log_prob(self, x: Tensor) -> Tensor: - """the log probability density function of the uniform noise. + """The log probability density function of the uniform noise. Args: x (Tensor): the input tensor. @@ -153,7 +154,7 @@ def log_prob(self, x: Tensor) -> Tensor: return torch.log(lb.mul(ub)) - torch.log(self.high - self.low) def cdf(self, x: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """the cumulative distribution function of the uniform noise. + """The cumulative distribution function of the uniform noise. Args: x (TENSOR_OPERABLE): the input tensor. @@ -165,7 +166,7 @@ def cdf(self, x: TENSOR_OPERABLE) -> TENSOR_OPERABLE: return result.clamp(min=0, max=1) def forward(self, x: Tensor) -> Tensor: - """add the uniform noise to the input tensor. + """Add the uniform noise to the input tensor. Args: x (Tensor): the input tensor. @@ -176,7 +177,7 @@ def forward(self, x: Tensor) -> Tensor: return torch.distributions.Uniform(low=x + self.low, high=x + self.high).sample() def extra_repr(self) -> str: - """the extra representation of the uniform noise. + """The extra representation of the uniform noise. Returns: str: the extra representation of the uniform noise. diff --git a/analogvnn/nn/normalize/Clamp.py b/analogvnn/nn/normalize/Clamp.py index 1f3afbb..7d96e56 100644 --- a/analogvnn/nn/normalize/Clamp.py +++ b/analogvnn/nn/normalize/Clamp.py @@ -9,12 +9,11 @@ class Clamp(Normalize): - """Implements the clamp normalization function with range [-1, 1]. - """ + """Implements the clamp normalization function with range [-1, 1].""" @staticmethod def forward(x: Tensor): - """forward pass of the clamp normalization function with range [-1, 1]. + """Forward pass of the clamp normalization function with range [-1, 1]. Args: x (Tensor): the input tensor. @@ -25,7 +24,7 @@ def forward(x: Tensor): return torch.clamp(x, min=-1, max=1) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the clamp normalization function with range [-1, 1]. + """Backward pass of the clamp normalization function with range [-1, 1]. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. @@ -39,12 +38,11 @@ def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: class Clamp01(Normalize): - """Implements the clamp normalization function with range [0, 1]. - """ + """Implements the clamp normalization function with range [0, 1].""" @staticmethod def forward(x: Tensor): - """forward pass of the clamp normalization function with range [0, 1]. + """Forward pass of the clamp normalization function with range [0, 1]. Args: x (Tensor): the input tensor. @@ -55,7 +53,7 @@ def forward(x: Tensor): return torch.clamp(x, min=0, max=1) def backward(self, grad_output: Optional[Tensor]) -> Optional[Tensor]: - """backward pass of the clamp normalization function with range [0, 1]. + """Backward pass of the clamp normalization function with range [0, 1]. Args: grad_output (Optional[Tensor]): the gradient of the output tensor. diff --git a/analogvnn/nn/normalize/LPNorm.py b/analogvnn/nn/normalize/LPNorm.py index 02b9c42..27bcce0 100644 --- a/analogvnn/nn/normalize/LPNorm.py +++ b/analogvnn/nn/normalize/LPNorm.py @@ -13,12 +13,13 @@ class LPNorm(Normalize): p (int): the pth power of the Lp norm. make_max_1 (bool): if True, the maximum absolute value of the output tensor will be 1. """ + __constants__ = ['p', 'make_max_1'] p: nn.Parameter make_max_1: nn.Parameter def __init__(self, p: int, make_max_1=False): - """initializes the row-wise Lp normalization function. + """Initializes the row-wise Lp normalization function. Args: p (int): the pth power of the Lp norm. @@ -28,8 +29,8 @@ def __init__(self, p: int, make_max_1=False): self.p = nn.Parameter(torch.tensor(p), requires_grad=False) self.make_max_1 = nn.Parameter(torch.tensor(make_max_1), requires_grad=False) - def forward(self, x: Tensor): - """forward pass of row-wise Lp normalization function. + def forward(self, x: Tensor) -> Tensor: + """Forward pass of row-wise Lp normalization function. Args: x (Tensor): the input tensor. @@ -52,10 +53,17 @@ def forward(self, x: Tensor): class LPNormW(LPNorm): - """Implements the whole matrix Lp normalization function. - """ + """Implements the whole matrix Lp normalization function.""" + + def forward(self, x: Tensor) -> Tensor: + """Forward pass of whole matrix Lp normalization function. - def forward(self, x: Tensor): + Args: + x (Tensor): the input tensor. + + Returns: + Tensor: the output tensor. + """ norm = torch.norm(x, self.p) norm = torch.clamp(norm, min=1e-4) x = torch.div(x, norm) @@ -67,64 +75,64 @@ def forward(self, x: Tensor): class L1Norm(LPNorm): - """Implements the row-wise L1 normalization function. - """ + """Implements the row-wise L1 normalization function.""" def __init__(self): + """Initializes the row-wise L1 normalization function.""" super(L1Norm, self).__init__(p=1, make_max_1=False) class L2Norm(LPNorm): - """Implements the row-wise L2 normalization function. - """ + """Implements the row-wise L2 normalization function.""" def __init__(self): + """Initializes the row-wise L2 normalization function.""" super(L2Norm, self).__init__(p=2, make_max_1=False) class L1NormW(LPNormW): - """Implements the whole matrix L1 normalization function. - """ + """Implements the whole matrix L1 normalization function.""" def __init__(self): + """Initializes the whole matrix L1 normalization function.""" super(L1NormW, self).__init__(p=1, make_max_1=False) class L2NormW(LPNormW): - """Implements the whole matrix L2 normalization function. - """ + """Implements the whole matrix L2 normalization function.""" def __init__(self): + """Initializes the whole matrix L2 normalization function.""" super(L2NormW, self).__init__(p=2, make_max_1=False) class L1NormM(LPNorm): - """Implements the row-wise L1 normalization function with maximum absolute value of 1. - """ + """Implements the row-wise L1 normalization function with maximum absolute value of 1.""" def __init__(self): + """Initializes the row-wise L1 normalization function with maximum absolute value of 1.""" super(L1NormM, self).__init__(p=1, make_max_1=True) class L2NormM(LPNorm): - """Implements the row-wise L2 normalization function with maximum absolute value of 1. - """ + """Implements the row-wise L2 normalization function with maximum absolute value of 1.""" def __init__(self): + """Initializes the row-wise L2 normalization function with maximum absolute value of 1.""" super(L2NormM, self).__init__(p=2, make_max_1=True) class L1NormWM(LPNormW): - """Implements the whole matrix L1 normalization function with maximum absolute value of 1. - """ + """Implements the whole matrix L1 normalization function with maximum absolute value of 1.""" def __init__(self): + """Initializes the whole matrix L1 normalization function with maximum absolute value of 1.""" super(L1NormWM, self).__init__(p=1, make_max_1=True) class L2NormWM(LPNormW): - """Implements the whole matrix L2 normalization function with maximum absolute value of 1. - """ + """Implements the whole matrix L2 normalization function with maximum absolute value of 1.""" def __init__(self): + """Initializes the whole matrix L2 normalization function with maximum absolute value of 1.""" super(L2NormWM, self).__init__(p=2, make_max_1=True) diff --git a/analogvnn/nn/normalize/Normalize.py b/analogvnn/nn/normalize/Normalize.py index 4929fd4..ea547cf 100644 --- a/analogvnn/nn/normalize/Normalize.py +++ b/analogvnn/nn/normalize/Normalize.py @@ -5,6 +5,4 @@ class Normalize(Layer, BackwardIdentity): - """This class is base class for all normalization functions. - """ - pass + """This class is base class for all normalization functions.""" diff --git a/analogvnn/nn/precision/ReducePrecision.py b/analogvnn/nn/precision/ReducePrecision.py index b674b29..3b5270e 100644 --- a/analogvnn/nn/precision/ReducePrecision.py +++ b/analogvnn/nn/precision/ReducePrecision.py @@ -17,12 +17,13 @@ class ReducePrecision(Layer, BackwardIdentity): divide (nn.Parameter): the rounding value that is if divide is 0.5, then 0.6 will be rounded to 1.0 and 0.4 will be rounded to 0.0. """ + __constants__ = ['precision', 'divide'] precision: nn.Parameter divide: nn.Parameter def __init__(self, precision: int = None, divide: float = 0.5): - """initialize the reduce precision function. + """Initialize the reduce precision function. Args: precision (int): the precision of the output tensor. @@ -31,20 +32,20 @@ def __init__(self, precision: int = None, divide: float = 0.5): """ super(ReducePrecision, self).__init__() if precision < 1: - raise ValueError(f"precision has to be more than 0, but got {precision}") + raise ValueError(f'precision has to be more than 0, but got {precision}') if precision != int(precision): - raise ValueError(f"precision must be int, but got {precision}") + raise ValueError(f'precision must be int, but got {precision}') if not (0 <= divide <= 1): - raise ValueError(f"divide must be between 0 and 1, but got {divide}") + raise ValueError(f'divide must be between 0 and 1, but got {divide}') self.precision = nn.Parameter(torch.tensor(precision), requires_grad=False) self.divide = nn.Parameter(torch.tensor(divide), requires_grad=False) @property def precision_width(self) -> Tensor: - """the precision width + """The precision width. Returns: Tensor: the precision width @@ -53,7 +54,7 @@ def precision_width(self) -> Tensor: @property def bit_precision(self) -> Tensor: - """the bit precision of the ReducePrecision module. + """The bit precision of the ReducePrecision module. Returns: Tensor: the bit precision of the ReducePrecision module. @@ -62,7 +63,7 @@ def bit_precision(self) -> Tensor: @staticmethod def convert_to_precision(bit_precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """convert the bit precision to the precision. + """Convert the bit precision to the precision. Args: bit_precision (TENSOR_OPERABLE): the bit precision. @@ -73,7 +74,7 @@ def convert_to_precision(bit_precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: return 2 ** bit_precision - 1 def extra_repr(self) -> str: - """the extra __repr__ string of the ReducePrecision module. + """The extra __repr__ string of the ReducePrecision module. Returns: str: string @@ -81,7 +82,7 @@ def extra_repr(self) -> str: return f'precision={int(self.precision)}, divide={float(self.divide):0.2f}' def forward(self, x: Tensor) -> Tensor: - """forward function of the ReducePrecision module. + """Forward function of the ReducePrecision module. Args: x (Tensor): the input tensor. diff --git a/analogvnn/nn/precision/StochasticReducePrecision.py b/analogvnn/nn/precision/StochasticReducePrecision.py index ef4ab18..7e36ae9 100644 --- a/analogvnn/nn/precision/StochasticReducePrecision.py +++ b/analogvnn/nn/precision/StochasticReducePrecision.py @@ -15,27 +15,28 @@ class StochasticReducePrecision(Layer, BackwardIdentity): Attributes: precision (nn.Parameter): the precision of the output tensor. """ + __constants__ = ['precision'] precision: nn.Parameter def __init__(self, precision: int = 8): - """initialize the StochasticReducePrecision module. + """Initialize the StochasticReducePrecision module. Args: precision (int): the precision of the output tensor. """ super(StochasticReducePrecision, self).__init__() if precision < 1: - raise ValueError("precision has to be more than 0, but got {}".format(precision)) + raise ValueError('precision has to be more than 0, but got {}'.format(precision)) if precision != int(precision): - raise ValueError("precision must be int, but got {}".format(precision)) + raise ValueError('precision must be int, but got {}'.format(precision)) self.precision = nn.Parameter(torch.tensor(precision), requires_grad=False) @property def precision_width(self) -> Tensor: - """the precision width + """The precision width. Returns: Tensor: the precision width @@ -44,7 +45,7 @@ def precision_width(self) -> Tensor: @property def bit_precision(self) -> Tensor: - """the bit precision of the ReducePrecision module. + """The bit precision of the ReducePrecision module. Returns: Tensor: the bit precision of the ReducePrecision module. @@ -53,7 +54,7 @@ def bit_precision(self) -> Tensor: @staticmethod def convert_to_precision(bit_precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: - """convert the bit precision to the precision. + """Convert the bit precision to the precision. Args: bit_precision (TENSOR_OPERABLE): the bit precision. @@ -64,7 +65,7 @@ def convert_to_precision(bit_precision: TENSOR_OPERABLE) -> TENSOR_OPERABLE: return 2 ** bit_precision - 1 def extra_repr(self) -> str: - """the extra __repr__ string of the StochasticReducePrecision module. + """The extra __repr__ string of the StochasticReducePrecision module. Returns: str: string @@ -72,7 +73,7 @@ def extra_repr(self) -> str: return f'precision={self.precision}' def forward(self, x: Tensor) -> Tensor: - """forward function of the StochasticReducePrecision module. + """Forward function of the StochasticReducePrecision module. Args: x (Tensor): input tensor. diff --git a/analogvnn/parameter/PseudoParameter.py b/analogvnn/parameter/PseudoParameter.py index c97c5ee..0d7b7bf 100644 --- a/analogvnn/parameter/PseudoParameter.py +++ b/analogvnn/parameter/PseudoParameter.py @@ -20,6 +20,7 @@ class PseudoParameterModule(nn.Module): original (PseudoParameter): the original parameters. _transformed (nn.Parameter): the transformed parameters. """ + original: PseudoParameter _transformed: nn.Parameter @@ -30,7 +31,7 @@ def __init__(self, original, transformed): original (PseudoParameter): the original parameters. transformed (nn.Parameter): the transformed parameters. """ - super().__init__() + super(PseudoParameterModule, self).__init__() self.original = original self._transformed = transformed @@ -53,7 +54,7 @@ def __call__(self, *args, **kwargs) -> nn.Parameter: """Alias for __call__""" def set_original_data(self, data: Tensor) -> PseudoParameterModule: - """set data to the original parameter. + """Set data to the original parameter. Args: data (Tensor): the data to set. @@ -92,6 +93,7 @@ class PseudoParameter(Parameter): module (PseudoParameterModule): the module that wraps the parameter and the transformation. transformation (Callable): the transformation. """ + _transformation: Callable _transformed: nn.Parameter _module: PseudoParameterModule @@ -118,7 +120,7 @@ def __init__(self, data=None, requires_grad=True, transformation=None, *args, ** *args: additional arguments. **kwargs: additional keyword arguments. """ - super().__init__(data, requires_grad, *args, **kwargs) + super(PseudoParameter, self).__init__(data, requires_grad, *args, **kwargs) self._transformed = nn.Parameter(data=data, requires_grad=requires_grad) self._transformed.original = self self._transformation = self.identity @@ -145,7 +147,7 @@ def __call__(self, *args, **kwargs): try: self._transformed.data = self._transformation(self) except Exception as e: - raise RuntimeError(f"here: {e.args}") from e + raise RuntimeError(f'here: {e.args}') from e return self._transformed def __repr__(self): @@ -260,7 +262,7 @@ def parametrize_module(cls, module: nn.Module, transformation: Callable, require for sub_module in module.children(): if sub_module == module: continue - if hasattr(module, "parametrizations") and ( + if hasattr(module, 'parametrizations') and ( sub_module is module.parametrizations or sub_module in module.parametrizations ): continue diff --git a/analogvnn/utils/TensorboardModelLog.py b/analogvnn/utils/TensorboardModelLog.py index 15c9b66..13f08e1 100644 --- a/analogvnn/utils/TensorboardModelLog.py +++ b/analogvnn/utils/TensorboardModelLog.py @@ -25,6 +25,7 @@ class TensorboardModelLog: layer_data (bool): whether to log the layer data. _log_record (Dict[str, bool]): the log record. """ + model: nn.Module tensorboard: Optional[SummaryWriter] layer_data: bool @@ -37,6 +38,7 @@ def __init__(self, model: Model, log_dir: str): model (nn.Module): the model to log. log_dir (str): the directory to log to. """ + super(TensorboardModelLog, self).__init__() self.model = model self.tensorboard = None self.layer_data = True @@ -46,7 +48,7 @@ def __init__(self, model: Model, log_dir: str): os.mkdir(log_dir) self.set_log_dir(log_dir) - if hasattr(model, "subscribe_tensorboard"): + if hasattr(model, 'subscribe_tensorboard'): model.subscribe_tensorboard(tensorboard=self) def set_log_dir(self, log_dir: str) -> TensorboardModelLog: @@ -61,17 +63,16 @@ def set_log_dir(self, log_dir: str) -> TensorboardModelLog: Raises: ValueError: if the log directory is invalid. """ - # https://github.com/tensorflow/tensorboard/pull/6135 from tensorboard.compat import tf - if getattr(tf, "io", None) is None: + if getattr(tf, 'io', None) is None: import tensorboard.compat.tensorflow_stub as new_tf tf.__dict__.update(new_tf.__dict__) if os.path.isdir(log_dir): self.tensorboard = SummaryWriter(log_dir=log_dir) else: - raise ValueError(f"Log directory {log_dir} does not exist.") + raise ValueError(f'Log directory {log_dir} does not exist.') return self def _add_layer_data(self, epoch: int = None): @@ -80,16 +81,11 @@ def _add_layer_data(self, epoch: int = None): Args: epoch (int): the epoch to add the data for. """ - idx = 0 - for module in self.model.modules(): - if isinstance(module, nn.Sequential) or isinstance(module, nn.ModuleList) or (module == self): + for name, parameter in self.model.named_parameters(): + if not parameter.requires_grad: continue - idx += 1 - if hasattr(module, "bias") and hasattr(module.bias, "size"): - self.tensorboard.add_histogram(f"{idx}-{module}.bias", module.bias, epoch) - if hasattr(module, "weight") and hasattr(module.weight, "size"): - self.tensorboard.add_histogram(f"{idx}-{module}.weight", module.weight, epoch) + self.tensorboard.add_histogram(name, parameter.data, epoch) def on_compile(self, layer_data: bool = True): """Called when the model is compiled. @@ -123,7 +119,7 @@ def add_graph( if model is None: model = self.model - log_id = f"{self.tensorboard.log_dir}_{TensorboardModelLog.add_graph.__name__}_{id(model)}" + log_id = f'{self.tensorboard.log_dir}_{TensorboardModelLog.add_graph.__name__}_{id(model)}' if log_id in self._log_record: return self @@ -136,7 +132,7 @@ def add_graph( use_autograd_graph = model.use_autograd_graph model.use_autograd_graph = False - graph_path = Path(self.tensorboard.log_dir).joinpath(f"graph_{model.__class__.__name__}_{id(model)}") + graph_path = Path(self.tensorboard.log_dir).joinpath(f'graph_{model.__class__.__name__}_{id(model)}') with SummaryWriter(log_dir=str(graph_path)) as graph_writer: graph_writer.add_graph(model, torch.zeros(input_size).to(model.device)) @@ -165,16 +161,15 @@ def add_summary( Raises: ImportError: if torchinfo (https://github.com/tyleryep/torchinfo) is not installed. """ - try: import torchinfo except ImportError as e: - raise ImportError("requires torchinfo: https://github.com/tyleryep/torchinfo") from e + raise ImportError('requires torchinfo: https://github.com/tyleryep/torchinfo') from e if model is None: model = self.model - log_id = f"{self.tensorboard.log_dir}_{TensorboardModelLog.add_summary.__name__}_{id(model)}" + log_id = f'{self.tensorboard.log_dir}_{TensorboardModelLog.add_summary.__name__}_{id(model)}' if input_size is None: data_shape = next(iter(train_loader))[0].shape @@ -198,18 +193,18 @@ def add_summary( nn_model_summary.formatting.verbose = torchinfo.Verbosity.VERBOSE model_str = str(model) - nn_model_summary = f"{nn_model_summary}" + nn_model_summary = f'{nn_model_summary}' if log_id in self._log_record: return model_str, nn_model_summary self.tensorboard.add_text( - f"str ({model.__class__.__name__})", - re.sub("\n", "\n ", f" {model_str}") + f'str ({model.__class__.__name__})', + re.sub('\n', '\n ', f' {model_str}') ) self.tensorboard.add_text( - f"summary ({model.__class__.__name__})", - re.sub("\n", "\n ", f" {nn_model_summary}") + f'summary ({model.__class__.__name__})', + re.sub('\n', '\n ', f' {nn_model_summary}') ) self._log_record[log_id] = True return model_str, nn_model_summary @@ -226,7 +221,7 @@ def register_training(self, epoch: int, train_loss: float, train_accuracy: float TensorboardModelLog: self. """ self.tensorboard.add_scalar('Loss/train', train_loss, epoch) - self.tensorboard.add_scalar("Accuracy/train", train_accuracy, epoch) + self.tensorboard.add_scalar('Accuracy/train', train_accuracy, epoch) if self.layer_data: self._add_layer_data(epoch=epoch) return self @@ -243,12 +238,16 @@ def register_testing(self, epoch: int, test_loss: float, test_accuracy: float) - TensorboardModelLog: self. """ self.tensorboard.add_scalar('Loss/test', test_loss, epoch) - self.tensorboard.add_scalar("Accuracy/test", test_accuracy, epoch) + self.tensorboard.add_scalar('Accuracy/test', test_accuracy, epoch) return self # noinspection PyUnusedLocal def close(self, *args, **kwargs): """Close the tensorboard. + + Args: + *args: ignored. + **kwargs: ignored. """ if self.tensorboard is not None: self.tensorboard.close() diff --git a/analogvnn/utils/common_types.py b/analogvnn/utils/common_types.py index ef812e6..60028e4 100644 --- a/analogvnn/utils/common_types.py +++ b/analogvnn/utils/common_types.py @@ -5,10 +5,10 @@ __all__ = ['TENSOR_OPERABLE', 'TENSOR_CALLABLE', 'TENSORS'] TENSOR_OPERABLE = Union[Sequence[Tensor], Tensor, int, float, bool] -""" `TENSOR_OPERABLE` is a type alias for types that can be operated on by a tensor. """ +"""`TENSOR_OPERABLE` is a type alias for types that can be operated on by a tensor. """ TENSOR_CALLABLE = Callable[[TENSOR_OPERABLE], TENSOR_OPERABLE] -""" `TENSOR_CALLABLE` is a type alias for a function that takes a `TENSOR_OPERABLE` and returns a `TENSOR_OPERABLE`. """ +"""`TENSOR_CALLABLE` is a type alias for a function that takes a `TENSOR_OPERABLE` and returns a `TENSOR_OPERABLE`. """ TENSORS = Union[None, Tensor, Sequence[Tensor]] -""" `TENSORS` is a type alias for a tensor or a sequence of tensors. """ +"""`TENSORS` is a type alias for a tensor or a sequence of tensors. """ diff --git a/analogvnn/utils/is_cpu_cuda.py b/analogvnn/utils/is_cpu_cuda.py index 7a9e820..9c204fc 100644 --- a/analogvnn/utils/is_cpu_cuda.py +++ b/analogvnn/utils/is_cpu_cuda.py @@ -14,11 +14,13 @@ class CPUCuda: _device (torch.device): The device. device_name (str): The name of the device. """ + _device: torch.device device_name: str def __init__(self): """Initialize the CPUCuda class.""" + super(CPUCuda, self).__init__() self._device = None self.device_name = None self.reset_device() @@ -29,7 +31,7 @@ def reset_device(self): Returns: CPUCuda: self """ - self.set_device(f"cuda:{torch.cuda.current_device()}" if torch.cuda.is_available() else "cpu") + self.set_device(f'cuda:{torch.cuda.current_device()}' if torch.cuda.is_available() else 'cpu') return self def set_device(self, device_name: str) -> CPUCuda: @@ -41,7 +43,6 @@ def set_device(self, device_name: str) -> CPUCuda: Returns: CPUCuda: self """ - self._device = torch.device(device_name) self.device_name = self._device.type return self @@ -53,7 +54,7 @@ def is_cuda(self) -> bool: Returns: bool: True if the device is cuda, False otherwise. """ - return "cuda" in self.device_name + return 'cuda' in self.device_name @property def device(self) -> torch.device: @@ -82,10 +83,9 @@ def get_module_device(self, module) -> torch.device: Returns: torch.device: the device of the module. """ - # noinspection PyBroadException try: - device: torch.device = getattr(module, "device", None) + device: torch.device = getattr(module, 'device', None) if device is None: device = next(module.parameters()).device return device diff --git a/analogvnn/utils/render_autograd_graph.py b/analogvnn/utils/render_autograd_graph.py index 0cea22d..e6f9f65 100644 --- a/analogvnn/utils/render_autograd_graph.py +++ b/analogvnn/utils/render_autograd_graph.py @@ -14,6 +14,7 @@ from torch import Tensor, nn from torch.nn import Parameter +from analogvnn.backward.BackwardModule import BackwardModule from analogvnn.nn.module.Layer import Layer from analogvnn.utils.is_cpu_cuda import is_cpu_cuda @@ -23,7 +24,7 @@ __all__ = [ 'size_to_str', 'AutoGradDot', - 'make_autograd_obj_from_output', + 'make_autograd_obj_from_outputs', 'make_autograd_obj_from_module', 'get_autograd_dot_from_outputs', 'get_autograd_dot_from_module', @@ -36,7 +37,7 @@ Node = namedtuple('Node', ('name', 'inputs', 'attr', 'op')) # Saved attrs for grad_fn (incl. saved variables) begin with `._saved_*` -SAVED_PREFIX = "_saved_" +SAVED_PREFIX = '_saved_' def size_to_str(size): @@ -51,6 +52,10 @@ def size_to_str(size): return '(' + ', '.join(['%d' % s for s in size]) + ')' +def _format_name_size(name, size): + return '%s\n %s' % (name, size) + + def resize_graph(dot: Digraph, size_per_element: float = 0.15, min_size: float = 12): """Resize the graph according to how much content it contains. @@ -66,7 +71,7 @@ def resize_graph(dot: Digraph, size_per_element: float = 0.15, min_size: float = num_rows = len(dot.body) content_size = num_rows * size_per_element size = max(min_size, content_size) - size_str = str(size) + "," + str(size) + size_str = str(size) + ',' + str(size) dot.graph_attr.update(size=size_str) @@ -81,22 +86,21 @@ def get_fn_name(fn: Callable, show_attrs: bool, max_attr_chars: int) -> str: Returns: str: the name of the function. """ - name = str(type(fn).__name__) if name.endswith('Backward'): name = name[:-8] if not show_attrs: return name - attrs = dict() + attrs = {} for attr in dir(fn): if not attr.startswith(SAVED_PREFIX): continue val = getattr(fn, attr) attr = attr[len(SAVED_PREFIX):] if torch.is_tensor(val): - attrs[attr] = "[saved tensor]" + attrs[attr] = '[saved tensor]' elif isinstance(val, Sequence) and any(torch.is_tensor(t) for t in val): - attrs[attr] = "[saved tensors]" + attrs[attr] = '[saved tensors]' else: attrs[attr] = str(val) if not attrs: @@ -104,9 +108,9 @@ def get_fn_name(fn: Callable, show_attrs: bool, max_attr_chars: int) -> str: max_attr_chars = max(max_attr_chars, 3) col1width = max(len(k) for k in attrs.keys()) col2width = min(max(len(str(v)) for v in attrs.values()), max_attr_chars) - sep = "-" * max(col1width + col2width + 2, len(name)) + sep = '-' * max(col1width + col2width + 2, len(name)) attrstr = '%-' + str(col1width) + 's: %' + str(col2width) + 's' - truncate = lambda s: s[:col2width - 3] + "..." if len(s) > col2width else s + truncate = lambda s: s[:col2width - 3] + '...' if len(s) > col2width else s # noqa: E731 params = '\n'.join(attrstr % (k, truncate(str(v))) for (k, v) in attrs.items()) return name + '\n' + sep + '\n' + params @@ -147,6 +151,7 @@ class AutoGradDot: show_saved: bool = dataclasses.field(default=False, repr=False, hash=False) max_attr_chars: int = dataclasses.field(default=50, repr=False, hash=False) _called: bool = False + _ignore_tensor: Dict[int, bool] = dataclasses.field(default_factory=dict, repr=False, hash=False) def __post_init__(self): """Create the graphviz graph. @@ -154,26 +159,27 @@ def __post_init__(self): Raises: ImportError: if graphviz (https://pygraphviz.github.io/) is not available. """ - try: from graphviz import Digraph except ImportError as e: - raise ImportError("requires graphviz: https://pygraphviz.github.io/") from e - - node_attr = dict( - style='filled', - shape='box', - align='left', - fontsize='12', - ranksep='0.1', - height='0.2', - fontname='monospace' - ) - self.dot = Digraph(node_attr=node_attr, graph_attr=dict(size="12,12"), format="svg") + raise ImportError('requires graphviz: https://pygraphviz.github.io/') from e + + node_attr = { + 'style': 'filled', + 'shape': 'box', + 'align': 'left', + 'fontsize': '12', + 'ranksep': '0.1', + 'height': '0.2', + 'fontname': 'monospace' + } + self.dot = Digraph(node_attr=node_attr, graph_attr={'size': '12,12'}, format='svg') + # noinspection PyProtectedMember + self.add_ignore_tensor(BackwardModule._empty_holder_tensor) @property def inputs(self) -> Sequence[Tensor]: - """the arg inputs to the module + """The arg inputs to the module. Returns: Sequence[Tensor]: the arg inputs to the module. @@ -197,12 +203,12 @@ def inputs(self, inputs: Union[Tensor, Sequence[Tensor]]): inputs = (inputs,) for i, v in enumerate(inputs): - self.param_map[id(v)] = f"INPUT_{i}" - self.param_map[id(v.data)] = f"INPUT_{i}" + self.param_map[id(v)] = f'INPUT_{i}' + self.param_map[id(v.data)] = f'INPUT_{i}' @property def inputs_kwargs(self) -> Dict[str, Tensor]: - """the keyword inputs to the module + """The keyword inputs to the module. Args: Dict[str, Tensor]: the keyword inputs to the module. @@ -223,12 +229,12 @@ def inputs_kwargs(self, inputs_kwargs: Dict[str, Tensor]): return for k, v in inputs_kwargs.items(): - self.param_map[id(v)] = f"INPUT_{k}" - self.param_map[id(v.data)] = f"INPUT_{k}" + self.param_map[id(v)] = f'INPUT_{k}' + self.param_map[id(v.data)] = f'INPUT_{k}' @property def outputs(self) -> Optional[Sequence[Tensor]]: - """the outputs of the module + """The outputs of the module. Returns: Optional[Sequence[Tensor]]: the outputs of the module. @@ -243,12 +249,12 @@ def outputs(self, outputs): self._outputs = outputs for i, v in enumerate(outputs): - self.param_map[id(v)] = f"OUTPUT_{i}" - self.param_map[id(v.data)] = f"OUTPUT_{i}" + self.param_map[id(v)] = f'OUTPUT_{i}' + self.param_map[id(v.data)] = f'OUTPUT_{i}' @property - def module(self): - """the module. + def module(self) -> nn.Module: + """The module. Returns: nn.Module: the module to be traced. @@ -286,7 +292,40 @@ def reset_params(self): self.module = self.module return self - def get_tensor_name(self, tensor: Tensor, name: Optional[str] = None): + @property + def ignore_tensor(self) -> Dict[int, bool]: + """The tensor ignored from the dot graphs. + + Returns: + Dict[int, bool]: the ignore tensor dict. + """ + return self._ignore_tensor + + def add_ignore_tensor(self, tensor: Tensor): + """Add a tensor to the ignore tensor dict. + + Args: + tensor (Tensor): the tensor to ignore. + + Returns: + AutoGradDot: self. + """ + self._ignore_tensor[id(tensor)] = True + return self + + def del_ignore_tensor(self, tensor: Tensor): + """Delete a tensor from the ignore tensor dict. + + Args: + tensor (Tensor): the tensor to delete. + + Returns: + AutoGradDot: self. + """ + self._ignore_tensor.pop(id(tensor), None) + return self + + def get_tensor_name(self, tensor: Tensor, name: Optional[str] = None) -> Tuple[str, str]: """Get the name of the tensor. Args: @@ -294,21 +333,22 @@ def get_tensor_name(self, tensor: Tensor, name: Optional[str] = None): name (Optional[str]): the name of the tensor. Defaults to None. Returns: - str: the name of the tensor. + Tuple[str, str]: the name and size of the tensor. """ if not name: if id(tensor) in self.param_map: name = self.param_map[id(tensor)] - elif hasattr(tensor, "name") and not not tensor.name: + elif hasattr(tensor, 'name') and not not tensor.name: name = tensor.name - elif hasattr(tensor, "names") and not not tensor.names: + elif hasattr(tensor, 'names') and not not tensor.names: if len(tensor.names) == 1: name = tensor.names[0] else: name = str(tensor.names) else: name = '' - return '%s\n %s' % (name, size_to_str(tensor.size())) + name, size = name.strip(), size_to_str(tensor.size()).strip() + return name, size def add_tensor(self, tensor: Tensor, name: Optional[str] = None, _attributes=None, **kwargs): """Add a tensor to the graph. @@ -325,7 +365,7 @@ def add_tensor(self, tensor: Tensor, name: Optional[str] = None, _attributes=Non self._seen.add(tensor) self.dot.node( name=str(id(tensor)), - label=self.get_tensor_name(tensor, name=name), + label=_format_name_size(*self.get_tensor_name(tensor, name=name)), _attributes=_attributes, **kwargs ) @@ -397,17 +437,34 @@ def is_seen(self, item: Any) -> bool: return item in self._seen -def add_grad_fn(grad_fn, autograd_dot: AutoGradDot): - """Add a grad_fn to the graph. +def _add_grad_fn(link: Union[Tensor, Callable], autograd_dot: AutoGradDot) -> Optional[List]: # noqa: C901 + """Add a link to the graph. Args: - grad_fn (Any): the Tensor.grad_fn. + link (Union[Tensor, Callable]): the Tensor or Tensor.grad_fn. autograd_dot (AutoGradDot): the AutoGradDot object. """ - assert not torch.is_tensor(grad_fn) - if autograd_dot.is_seen(grad_fn): - return + if autograd_dot.is_seen(link): + return None + + next_links = [] + + if isinstance(link, Tensor): + tensor = link + + autograd_dot.add_tensor(tensor, fillcolor='darkolivegreen1' if not tensor._is_view() else 'darkolivegreen3') + + if tensor.grad_fn: + next_links.append(tensor.grad_fn) + autograd_dot.add_edge(tensor.grad_fn, tensor) + + if tensor._is_view(): + next_links.append(tensor._base) + autograd_dot.add_edge(tensor._base, tensor, style='dotted') + return next_links + + grad_fn = link # add the node for this grad_fn autograd_dot.add_fn(grad_fn) @@ -421,7 +478,7 @@ def add_grad_fn(grad_fn, autograd_dot: AutoGradDot): attr = attr[len(SAVED_PREFIX):] if torch.is_tensor(val): - autograd_dot.add_edge(grad_fn, val, dir="none") + autograd_dot.add_edge(grad_fn, val, dir='none') autograd_dot.add_tensor(val, name=attr, fillcolor='orange') continue @@ -430,7 +487,7 @@ def add_grad_fn(grad_fn, autograd_dot: AutoGradDot): if not torch.is_tensor(t): continue name = attr + '[%s]' % str(i) - autograd_dot.add_edge(grad_fn, t, dir="none") + autograd_dot.add_edge(grad_fn, t, dir='none') autograd_dot.add_tensor(t, name=name, fillcolor='orange') if hasattr(grad_fn, 'variable'): @@ -448,52 +505,34 @@ def add_grad_fn(grad_fn, autograd_dot: AutoGradDot): if ( u[0].__class__.__name__ == 'AccumulateGrad' and hasattr(u[0], 'variable') and - autograd_dot.get_tensor_name(u[0].variable) == "_empty_holder_tensor\n (1)" + id(u[0].variable) in autograd_dot.ignore_tensor ): continue autograd_dot.add_edge(u[0], grad_fn) - add_grad_fn(u[0], autograd_dot=autograd_dot) + next_links.append(u[0]) # note: this used to show .saved_tensors in pytorch0.2, but stopped # working* as it was moved to ATen and Variable-Tensor merged # also note that this still works for custom autograd functions if hasattr(grad_fn, 'saved_tensors'): for t in grad_fn.saved_tensors: + if t is None: + continue autograd_dot.add_edge(t, grad_fn) autograd_dot.add_tensor(t, fillcolor='orange') + return next_links -def add_base_tensor(tensor: Tensor, autograd_dot: AutoGradDot, color: str = 'darkolivegreen1'): - """Add a base tensor to the graph. - - Args: - tensor (Tensor): the base tensor. - autograd_dot (AutoGradDot): the AutoGradDot object. - color (str): the color of the base tensor in graph. Defaults to 'darkolivegreen1'. - """ - if autograd_dot.is_seen(tensor): - return - - autograd_dot.add_tensor(tensor, fillcolor=color) - - if tensor.grad_fn: - add_grad_fn(tensor.grad_fn, autograd_dot=autograd_dot) - autograd_dot.add_edge(tensor.grad_fn, tensor) - - if tensor._is_view(): - add_base_tensor(tensor._base, autograd_dot=autograd_dot, color='darkolivegreen3') - autograd_dot.add_edge(tensor._base, tensor, style="dotted") - -def compile_autograd_obj( +def _compile_autograd_obj( autograd_dot: AutoGradDot, additional_params: Optional[dict] = None, show_attrs: bool = True, show_saved: bool = True, max_attr_chars: int = 50, ) -> AutoGradDot: - """Make dot graph in AutoGradDot + """Make dot graph in AutoGradDot. If a node represents a backward function, it is gray. Otherwise, the node represents a tensor and is either blue, orange, or green: @@ -519,11 +558,11 @@ def compile_autograd_obj( Returns: AutoGradDot: graphviz representation of autograd graph """ - if LooseVersion(torch.__version__) < LooseVersion("1.9") and (show_attrs or show_saved): + if LooseVersion(torch.__version__) < LooseVersion('1.9') and (show_attrs or show_saved): warnings.warn( - "make_dot: showing grad_fn attributes and saved variables" - " requires PyTorch version >= 1.9. (This does NOT apply to" - " saved tensors saved by custom autograd functions.)" + 'make_dot: showing grad_fn attributes and saved variables' + ' requires PyTorch version >= 1.9. (This does NOT apply to' + ' saved tensors saved by custom autograd functions.)' ) autograd_dot.show_attrs = show_attrs @@ -533,16 +572,35 @@ def compile_autograd_obj( if additional_params is not None: autograd_dot.param_map.update(additional_params) - # handle multiple outputs - for v in autograd_dot.outputs: - add_base_tensor(tensor=v, autograd_dot=autograd_dot) + deque = list(autograd_dot.outputs) + + while len(deque) > 0: + r = _add_grad_fn(deque.pop(0), autograd_dot=autograd_dot) + if r is not None: + deque += r resize_graph(autograd_dot.dot) return autograd_dot -def make_autograd_obj_from_output( +def _toggle_autograd_backward(disable, status, self): + if not isinstance(self, Layer): + return + + self = self.backward_function + + if self is None: + return + + if disable: + status[id(self)] = self._disable_autograd_backward + self._disable_autograd_backward = True + else: + self._disable_autograd_backward = status[id(self)] + + +def make_autograd_obj_from_outputs( outputs: Union[Tensor, Sequence[Tensor]], named_params: Union[Dict[str, Any], Iterator[Tuple[str, Parameter]]], additional_params: Optional[dict] = None, @@ -575,7 +633,7 @@ def make_autograd_obj_from_output( autograd_dot.param_map[id(v)] = k autograd_dot.param_map[id(v.data)] = k - return compile_autograd_obj(autograd_dot, additional_params, show_attrs, show_saved, max_attr_chars) + return _compile_autograd_obj(autograd_dot, additional_params, show_attrs, show_saved, max_attr_chars) def make_autograd_obj_from_module( @@ -607,7 +665,6 @@ def make_autograd_obj_from_module( Returns: AutoGradDot: graphviz representation of autograd graph """ - assert isinstance(module, nn.Module) new_args = [] new_kwargs = {} @@ -638,41 +695,26 @@ def make_autograd_obj_from_module( use_autograd_graph_status = module.use_autograd_graph module.use_autograd_graph = True - def toggle_autograd_backward(disable, status, self): - if not isinstance(self, Layer): - return - - self = self.backward_function - - if self is None: - return - - if disable: - status[id(self)] = self._disable_autograd_backward - self._disable_autograd_backward = True - else: - self._disable_autograd_backward = status[id(self)] - disable_autograd_backward_status = {} if from_forward: - module.apply(partial(toggle_autograd_backward, True, disable_autograd_backward_status)) + module.apply(partial(_toggle_autograd_backward, True, disable_autograd_backward_status)) module.train() autograd_dot.outputs = module(*new_args, **new_kwargs) module.train(training_status) if from_forward: - module.apply(partial(toggle_autograd_backward, False, disable_autograd_backward_status)) + module.apply(partial(_toggle_autograd_backward, False, disable_autograd_backward_status)) if isinstance(module, Layer): module.use_autograd_graph = use_autograd_graph_status - autograd_dot = compile_autograd_obj(autograd_dot, additional_params, show_attrs, show_saved, max_attr_chars) + autograd_dot = _compile_autograd_obj(autograd_dot, additional_params, show_attrs, show_saved, max_attr_chars) return autograd_dot -def join_scope_name(name: str, scope: Dict[str, str]) -> str: +def _join_scope_name(name: str, scope: Dict[str, str]) -> str: return '/'.join([scope[name], name]) @@ -693,11 +735,11 @@ def parse_trace_graph(graph) -> List[Node]: for n in graph.nodes(): attrs = {k: n[k] for k in n.attributeNames()} attrs = str(attrs).replace("'", ' ') - inputs = [join_scope_name(i.uniqueName(), scope) for i in n.inputs()] + inputs = [_join_scope_name(i.uniqueName(), scope) for i in n.inputs()] uname = next(n.outputs()).uniqueName() nodes.append(Node( - name=join_scope_name(uname, scope), + name=_join_scope_name(uname, scope), op=n.kind(), inputs=inputs, attr=attrs @@ -709,7 +751,7 @@ def parse_trace_graph(graph) -> List[Node]: scope[uname] = 'unused' nodes.append(Node( - name=join_scope_name(uname, scope), + name=_join_scope_name(uname, scope), op='Parameter', inputs=[], attr=str(n.type()) @@ -719,7 +761,7 @@ def parse_trace_graph(graph) -> List[Node]: def get_autograd_dot_from_trace(trace) -> Digraph: - """ Produces graphs of torch.jit.trace outputs + """Produces graphs of torch.jit.trace outputs. Example: >>> trace, = torch.jit.trace(model, args=(x,)) @@ -731,30 +773,31 @@ def get_autograd_dot_from_trace(trace) -> Digraph: Returns: graphviz.Digraph: the resulting graph. """ - try: from graphviz import Digraph except ImportError as e: - raise ImportError("requires graphviz: https://pygraphviz.github.io/") from e + raise ImportError('requires graphviz: https://pygraphviz.github.io/') from e # from tensorboardX - if LooseVersion(torch.__version__) >= LooseVersion("0.4.1"): + if LooseVersion(torch.__version__) >= LooseVersion('0.4.1'): torch.onnx._optimize_trace(trace, torch._C._onnx.OperatorExportTypes.ONNX_ATEN_FALLBACK) - elif LooseVersion(torch.__version__) >= LooseVersion("0.4"): + elif LooseVersion(torch.__version__) >= LooseVersion('0.4'): torch.onnx._optimize_trace(trace, False) else: torch.onnx._optimize_trace(trace) graph = trace.graph() list_of_nodes = parse_trace_graph(graph) - node_attr = dict(style='filled', - shape='box', - align='left', - fontsize='12', - ranksep='0.1', - height='0.2') + node_attr = { + 'style': 'filled', + 'shape': 'box', + 'align': 'left', + 'fontsize': '12', + 'ranksep': '0.1', + 'height': '0.2' + } - dot = Digraph(node_attr=node_attr, graph_attr=dict(size="12,12")) + dot = Digraph(node_attr=node_attr, graph_attr={'size': '12,12'}) for node in list_of_nodes: dot.node(node.name, label=node.name.replace('/', '\n')) @@ -792,7 +835,7 @@ def get_autograd_dot_from_outputs( Returns: Digraph: graphviz representation of autograd graph """ - return make_autograd_obj_from_output( + return make_autograd_obj_from_outputs( outputs=outputs, named_params=named_params, additional_params=additional_params, @@ -877,7 +920,7 @@ def save_autograd_graph_from_outputs( show_attrs=show_attrs, show_saved=show_saved, max_attr_chars=max_attr_chars, - ).render(filename, format="svg", cleanup=True) + ).render(filename, format='svg', cleanup=True) def save_autograd_graph_from_module( @@ -911,7 +954,6 @@ def save_autograd_graph_from_module( Returns: str: The (possibly relative) path of the rendered file. """ - return get_autograd_dot_from_module( module, *args, @@ -921,7 +963,7 @@ def save_autograd_graph_from_module( max_attr_chars=max_attr_chars, from_forward=from_forward, **kwargs - ).render(filename, format="svg", cleanup=True) + ).render(filename, format='svg', cleanup=True) def save_autograd_graph_from_trace(filename: Union[str, Path], trace) -> str: @@ -934,5 +976,4 @@ def save_autograd_graph_from_trace(filename: Union[str, Path], trace) -> str: Returns: str: The (possibly relative) path of the rendered file. """ - - return get_autograd_dot_from_trace(trace).render(filename, format="svg", cleanup=True) + return get_autograd_dot_from_trace(trace).render(filename, format='svg', cleanup=True) diff --git a/docs/_static/AnalogVNN_Demo.ipynb b/docs/_static/AnalogVNN_Demo.ipynb index 3b72b99..ee559f3 100644 --- a/docs/_static/AnalogVNN_Demo.ipynb +++ b/docs/_static/AnalogVNN_Demo.ipynb @@ -2,63 +2,67 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "id": "3wF5wszaj97Y" - }, - "source": [ - "# AnalogVNN Demo/Tutorial" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "04QgGZc9bF5D" - }, - "source": [ - "#### To create 3 layered linear photonic analog neural network with 4-bit [precision](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#reduceprecision), 0.2 [leakage](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#leakage-or-error-probability) and [clamp](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#clamp) normalization" - ] - }, - { - "cell_type": "markdown", + "metadata": {}, "source": [ - "Copyright 2021-present Vivswan Shah (vivswanshah@pitt.edu)\n", + "# AnalogVNN Demo/Tutorial\n", + "\n", + "Copyright © 2021-present Vivswan Shah (vivswanshah@pitt.edu)\n", + "\n", + "
\n", "\n", "[![arXiv](https://img.shields.io/badge/arXiv-2210.10048-orange.svg)](https://arxiv.org/abs/2210.10048)\n", "[![PyPI version](https://badge.fury.io/py/analogvnn.svg)](https://badge.fury.io/py/analogvnn)\n", "[![Documentation Status](https://readthedocs.org/projects/analogvnn/badge/?version=stable)](https://analogvnn.readthedocs.io/en/stable/?badge=stable)\n", "[![Python](https://img.shields.io/badge/python-3.7--3.10-blue)](https://badge.fury.io/py/analogvnn)\n", - "[![License: MPL 2.0](https://img.shields.io/badge/License-MPL_2.0-blue.svg)](https://opensource.org/licenses/MPL-2.0)\n" - ], - "metadata": { - "collapsed": false - } - }, - { - "cell_type": "markdown", - "source": [ + "[![License: MPL 2.0](https://img.shields.io/badge/License-MPL_2.0-blue.svg)](https://opensource.org/licenses/MPL-2.0)\n", + "\n", "\n", " \n", " \n", " \n", " \n", "
\n", - " View on AnalogVNN\n", + " \n", + "
\n", + " \n", + "
\n", + " View on AnalogVNN\n", + " \n", " 		\n", - " Run in Google Colab\n", + " \n", + "
\n", + " \n", + "
\n", + " Run in Google Colab\n", + " \n", " 		\n", - " View source on GitHub\n", + " \n", + "
\n", + " \n", + "
\n", + " View source on GitHub\n", + " \n", " 		\n", - " Download notebook\n", + " \n", + "
\n", + " \n", + "
\n", + " Download notebook\n", + " \n", "
" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "markdown", "source": [ - "![3 Layered Linear Photonic Analog Neural Network](analogvnn_model.png)" + "#### To create 3 layered linear photonic analog neural network with 4-bit [precision](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#reduceprecision), 0.5 [leakage](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#leakage-or-error-probability) and [clamp](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#clamp) normalization:\n", + "\n", + "![3 Layered Linear Photonic Analog Neural Network](analogvnn_model.png)\n", + "\n", + "Python file:\n", + "[Sample code](https://github.com/Vivswan/AnalogVNN/blob/v1.0.0/sample_code.py)\n", + "and\n", + "[Sample code with logs](https://github.com/Vivswan/AnalogVNN/blob/v1.0.0/sample_code_with_logs.py)" ], "metadata": { "collapsed": false @@ -66,24 +70,24 @@ }, { "cell_type": "markdown", - "metadata": { - "id": "nnrWf3PCEzXL" - }, "source": [ "## Setting up the Enviroment AnalogVNN" - ] + ], + "metadata": { + "collapsed": false + } }, { "cell_type": "code", - "source": [ - "# Install AnalogVNN with Pip\n", - "!pip install analogvnn" - ], + "execution_count": null, "metadata": { "id": "812kuN10TZgu" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "# Install AnalogVNN with Pip\n", + "!pip install analogvnn" + ] }, { "cell_type": "code", @@ -235,15 +239,20 @@ }, { "cell_type": "markdown", - "source": [ - "Note: [`analogvnn.nn.module.Sequential.Sequential.add_sequence()`](https://analogvnn.readthedocs.io/en/v1.0.0/autoapi/analogvnn/nn/module/Sequential/index.html#analogvnn.nn.module.Sequential.Sequential.add_sequence) is used to create and set forward and backward graphs in AnalogVNN, more information in Inner Workings" - ], "metadata": { "id": "iOkIKXWoZbmn" - } + }, + "source": [ + "Note: [`analogvnn.nn.module.Sequential.Sequential.add_sequence()`](https://analogvnn.readthedocs.io/en/v1.0.0/autoapi/analogvnn/nn/module/Sequential/index.html#analogvnn.nn.module.Sequential.Sequential.add_sequence) is used to create and set forward and backward graphs in AnalogVNN, more information in Inner Workings" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "39SIO0ICWiJ2" + }, + "outputs": [], "source": [ "nn_model = LinearModel(\n", " activation_class=GeLU,\n", @@ -251,15 +260,10 @@ " precision_class=ReducePrecision,\n", " precision=2 ** 4,\n", " noise_class=GaussianNoise,\n", - " leakage=0.2\n", + " leakage=0.5\n", ")\n", "print(nn_model)" - ], - "metadata": { - "id": "39SIO0ICWiJ2" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -276,6 +280,11 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tcN3WT8zWXQv" + }, + "outputs": [], "source": [ "class WeightModel(FullSequential):\n", " def __init__(self, norm_class, precision_class, precision, noise_class, leakage):\n", @@ -288,59 +297,54 @@ "\n", " self.eval()\n", " self.add_sequence(*self.all_layers)" - ], - "metadata": { - "id": "tcN3WT8zWXQv" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "Note: Since the `WeightModel` will only be used for converting the data to analog data to be used in the main `LinearModel`, we can use `eval()` to make sure the `WeightModel` is never been trained" - ], "metadata": { "id": "Dsudt6dXZBnV" - } + }, + "source": [ + "Note: Since the `WeightModel` will only be used for converting the data to analog data to be used in the main `LinearModel`, we can use `eval()` to make sure the `WeightModel` is never been trained" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FgehAu7qWlyV" + }, + "outputs": [], "source": [ "weight_model = WeightModel(\n", " norm_class=Clamp,\n", " precision_class=ReducePrecision,\n", " precision=2 ** 4,\n", " noise_class=GaussianNoise,\n", - " leakage=0.2\n", + " leakage=0.5\n", ")\n", "print(weight_model)" - ], - "metadata": { - "id": "FgehAu7qWlyV" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "Using [`PseudoParameter`](https://analogvnn.readthedocs.io/en/v1.0.0/inner_workings.html#pseudoparameters) to parametrize the parameter" - ], "metadata": { "id": "Dtg27Y80WwR0" - } + }, + "source": [ + "Using [`PseudoParameter`](https://analogvnn.readthedocs.io/en/v1.0.0/inner_workings.html#pseudoparameters) to parametrize the parameter" + ] }, { "cell_type": "code", - "source": [ - "PseudoParameter.parametrize_module(nn_model, transformation=weight_model)" - ], + "execution_count": null, "metadata": { "id": "O8i_yEZHWpZb" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "PseudoParameter.parametrize_module(nn_model, transformation=weight_model)" + ] }, { "cell_type": "markdown", @@ -353,30 +357,30 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bL9owooIXaJ2" + }, + "outputs": [], "source": [ "def cross_entropy_accuracy(output, target) -> float:\n", " _, preds = torch.max(output.data, 1)\n", " correct = (preds == target).sum().item()\n", " return correct / len(output)" - ], - "metadata": { - "id": "bL9owooIXaJ2" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "XnWd-loMXfLj" + }, + "outputs": [], "source": [ "nn_model.loss_function = nn.CrossEntropyLoss()\n", "nn_model.accuracy_function = cross_entropy_accuracy\n", "nn_model.optimizer = optim.Adam(params=nn_model.parameters())" - ], - "metadata": { - "id": "XnWd-loMXfLj" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -389,16 +393,16 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wbmpzLWpXo5t" + }, + "outputs": [], "source": [ "nn_model.compile(device=device)\n", "weight_model.compile(device=device)\n", "print(\"Compiled\")" - ], - "metadata": { - "id": "wbmpzLWpXo5t" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -411,6 +415,11 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9cAdBEaXX51G" + }, + "outputs": [], "source": [ "for epoch in range(10):\n", " train_loss, train_accuracy = nn_model.train_on(train_loader, epoch=epoch)\n", @@ -423,12 +432,7 @@ " f' Testing Loss: {test_loss:.4f},' \\\n", " f' Testing Accuracy: {100. * test_accuracy:.0f}%\\n'\n", " print(print_str)" - ], - "metadata": { - "id": "9cAdBEaXX51G" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -438,32 +442,45 @@ "source": [ "## Conclusion\n", "\n", - "Congratulations! You have trained a 3 layered linear photonic analog neural network with 4-bit [precision](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#reduceprecision), 0.2 [leakage](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#leakage-or-error-probability) and [clamp](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#clamp) normalization" + "Congratulations! You have trained a 3 layered linear photonic analog neural network with 4-bit [precision](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#reduceprecision), 0.5 [leakage](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#leakage-or-error-probability) and [clamp](https://analogvnn.readthedocs.io/en/v1.0.0/extra_classes.html#clamp) normalization" ] }, { "cell_type": "markdown", + "metadata": { + "id": "gic6z7KcYo1h" + }, "source": [ "GitHub: [https://github.com/Vivswan/AnalogVNN](https://github.com/Vivswan/AnalogVNN)\n", "\n", "Documentation: [https://analogvnn.readthedocs.io/](https://analogvnn.readthedocs.io/)" - ], - "metadata": { - "id": "gic6z7KcYo1h" - } + ] } ], "metadata": { + "accelerator": "GPU", "colab": { "provenance": [] }, + "gpuClass": "standard", "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", + "language": "python", "name": "python3" }, - "accelerator": "GPU", - "gpuClass": "standard" + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.15" + } }, "nbformat": 4, - "nbformat_minor": 0 + "nbformat_minor": 1 } diff --git a/docs/conf.py b/docs/conf.py index 8c85847..2b192a0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -79,7 +79,7 @@ exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] intersphinx_mapping = { - "python": ("https://docs.python.org/3/", None), + 'python': ('https://docs.python.org/3/', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), 'markdown_it': ('https://markdown-it-py.readthedocs.io/en/latest', None), 'numpy': ('https://numpy.org/doc/stable/', None), @@ -94,8 +94,7 @@ 'https://www.tensorflow.org/probability/api_docs/python', 'https://github.com/GPflow/tensorflow-intersphinx/raw/master/tfp_py_objects.inv' ), - "torch": ("https://pytorch.org/docs/stable/", None), - + 'torch': ('https://pytorch.org/docs/stable/', None), } intersphinx_disabled_domains = ['std'] @@ -126,7 +125,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = "en" +language = 'en' # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' diff --git a/docs/inner_workings.md b/docs/inner_workings.md index cd3fc67..ca2e1c9 100644 --- a/docs/inner_workings.md +++ b/docs/inner_workings.md @@ -16,12 +16,12 @@ layer to get parameterized data PyTorch's ParameterizedParameters vs AnalogVNN's PseudoParameters: - Similarity (Forward or Parameterizing the data): - > Data -> ParameterizingModel -> Parameterized Data + > Data → ParameterizingModel → Parameterized Data - Difference (Backward or Gradient Calculations): - ParameterizedParameters - > Parameterized Data -> ParameterizingModel -> Data - - PseudoParameters - > Parameterized Data -> Data + > Parameterized Data → ParameterizingModel → Data + - PseudoParameters + > Parameterized Data → Data So, by using `PseudoParameters` class the gradients of the parameter are calculated in such a way that the ParameterizingModel was never present. @@ -29,13 +29,13 @@ the ParameterizingModel was never present. To convert parameters of a layer or model to use PseudoParameters, then use: ```python - PseudoParameters.parameterize(Model, "parameter_name", transformation=ParameterizingModel) + PseudoParameters.parameterize(Model, "parameter_name", transformation=ParameterizingModel) ``` OR ```python - PseudoParameters.parametrize_module(Model, transformation=ParameterizingModel) + PseudoParameters.parametrize_module(Model, transformation=ParameterizingModel) ``` ## Forward and Backward Graphs diff --git a/docs/install.md b/docs/install.md index f5f541e..90d4f59 100644 --- a/docs/install.md +++ b/docs/install.md @@ -17,6 +17,9 @@ Install [PyTorch](https://pytorch.org/) then: ```bash # Current stable release for CPU and GPU pip install analogvnn + + # For additional optional features + pip install analogvnn[full] ``` OR diff --git a/docs/sample_code.md b/docs/sample_code.md index bb9b87a..bb827db 100644 --- a/docs/sample_code.md +++ b/docs/sample_code.md @@ -7,8 +7,11 @@ Run in Google Colab: ![3 Layered Linear Photonic Analog Neural Network](_static/analogvnn_model.png) -[Sample code](https://github.com/Photonics-Pitt-Org/AnalogVNN/blob/master/sample_code.py) for 3 layered linear photonic -analog neural network with 4-bit precision, 0.2 {ref}`extra_classes:leakage` and {ref}`extra_classes:clamp` +[Sample code](https://github.com/Vivswan/AnalogVNN/blob/v1.0.0/sample_code.py) +and +[Sample code with logs](https://github.com/Vivswan/AnalogVNN/blob/v1.0.0/sample_code_with_logs.py) +for 3 layered linear photonic analog neural network with 4-bit precision, +0.5 {ref}`extra_classes:leakage` and {ref}`extra_classes:clamp` normalization: ```{literalinclude} ../sample_code.py diff --git a/docs/tutorial.md b/docs/tutorial.md index 3d3ab39..5c4d7da 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -94,14 +94,14 @@ To convert a digital model to its analog counterpart the following steps needs t precision_class=ReducePrecision, precision=2 ** 4, noise_class=GaussianNoise, - leakage=0.2 + leakage=0.5 ) weight_model = WeightModel( norm_class=Clamp, precision_class=ReducePrecision, precision=2 ** 4, noise_class=GaussianNoise, - leakage=0.2 + leakage=0.5 ) # Setting Model Parameters diff --git a/pyproject.toml b/pyproject.toml index 658a7c3..3cfcf69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ name = "analogvnn" [project] # $ pip install analogvnn name = "analogvnn" -version = "1.0.0rc4" +version = "1.0.0rc5" description = "A fully modular framework for modeling and optimizing analog/photonic neural networks" # Optional readme = "README.md" requires-python = ">=3.7" @@ -72,24 +72,12 @@ dependencies = [ # Similar to `dependencies` above, these must be valid existing # projects. [project.optional-dependencies] -dev = [ - "setuptools>=61.0.0", - "build", # building the package {pyproject-build} - "twine", # to publish on pypi {twine upload --repository-url=https://test.pypi.org/legacy/ dist/*} {twine upload dist/*} - "johnnydep" # to see dependencies {johnnydep } -] -research = [ +full = [ "tensorflow", "tensorboard", - "torchviz", "torchinfo", - "matplotlib", - "tabulate", - "pillow", "graphviz", #"python-graphviz", - "seaborn", - "natsort", ] doc = [ "sphinx>=4.2.0", @@ -105,14 +93,30 @@ doc = [ "sphinxext-opengraph", "sphinxcontrib-katex", # for math ] -all = ["analogvnn[research,dev,doc]"] +flake8 = [ + "flake8", + "flake8-docstrings", + "flake8-quotes", + "flake8-bugbear", + "flake8-comprehensions", + "flake8-executable", + "flake8-coding", +] +dev = [ + "setuptools>=61.0.0", + "build", # building the package {pyproject-build} + "twine", # to publish on pypi {twine upload --repository-url=https://test.pypi.org/legacy/ dist/*} {twine upload dist/*} + "johnnydep", # to see dependencies {johnnydep } +] +test = ["analogvnn[flake8]"] +all = ["analogvnn[full,dev,doc,test]"] [project.urls] -"Homepage" = "https://github.com/Vivswan/AnalogVNN" +"Author" = "https://vivswan.github.io/" "Bug Reports" = "https://github.com/Vivswan/AnalogVNN/issues" "Documentation" = "https://analogvnn.readthedocs.io/en/latest/" -"Say Thanks!" = "https://github.com/Vivswan" +"Homepage" = "https://github.com/Vivswan/AnalogVNN" +"Say Thanks!" = "https://vivswan.github.io/" "Source" = "https://github.com/Vivswan/AnalogVNN" -"Author" = "https://vivswan.github.io/" # The following would provide a command line executable called `sample` # which executes the function `main` from this package when invoked. diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..2a15cd3 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +# Development +setuptools>=61.0.0 +build # building the package {pyproject-build} +twine # to publish on pypi {twine upload --repository-url=https://test.pypi.org/legacy/ dist/*} {twine upload dist/*} +johnnydep # to see dependencies {johnnydep } diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 0000000..b5ead38 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,13 @@ +# Docs +sphinx>=4.2.0 +sphinx-autobuild +rst-to-myst[sphinx] +myst_parser +furo +sphinx-rtd-theme +sphinx-autoapi +sphinx-copybutton +sphinx-notfound-page +sphinx-inline-tabs +sphinxext-opengraph +sphinxcontrib-katex # math diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..bba026e --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,8 @@ +# Testing +flake8 +flake8-docstrings +flake8-quotes +flake8-bugbear +flake8-comprehensions +flake8-executable +flake8-coding diff --git a/requirements.txt b/requirements.txt index 9d05a98..99fb2b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,36 +7,10 @@ scipy networkx importlib_metadata -# Development -setuptools>=61.0.0 -build # building the package {pyproject-build} -twine # to publish on pypi {twine upload --repository-url=https://test.pypi.org/legacy/ dist/*} {twine upload dist/*} -johnnydep # to see dependencies {johnnydep } - -# Research +# Full tensorflow tensorboard -torchviz torchinfo -matplotlib -tabulate pillow # conda install python-graphviz graphviz graphviz -# python-graphviz -seaborn -natsort - -# Docs -sphinx>=4.2.0 -sphinx-autobuild -rst-to-myst[sphinx] -myst_parser -furo -sphinx-rtd-theme -sphinx-autoapi -sphinx-copybutton -sphinx-notfound-page -sphinx-inline-tabs -sphinxext-opengraph -sphinxcontrib-katex # math diff --git a/research/__init__.py b/research/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/research/analog_vnn_1_analysis.py b/research/analog_vnn_1_analysis.py deleted file mode 100644 index 32347d7..0000000 --- a/research/analog_vnn_1_analysis.py +++ /dev/null @@ -1,176 +0,0 @@ -import argparse -import json -import os -from pathlib import Path -from typing import List - -import torch - - -def data_from_tensorboard(tensorboard_dir, destination: Path = None): - from tensorboard.plugins.hparams.metadata import SESSION_START_INFO_TAG - from tensorboard.plugins.hparams.plugin_data_pb2 import HParamsPluginData - from tensorflow.python.summary.summary_iterator import summary_iterator - - tensorboard_dir = Path(tensorboard_dir) - all_files: List[Path] = [] - parameter_data = {} - - for root, dirs, files in os.walk(tensorboard_dir): - for file in files: - all_files.append(Path(root).joinpath(file)) - - for i, file in enumerate(all_files): - name = file.parent - if "_" not in str(name.name): - name = name.parent - # c = True - name = name.name - - if i % 10 == 0: - print(f"[{i}/{len(all_files)}] Processing {name}...") - - if name not in parameter_data: - parameter_data[name] = { - "hyper_parameters": {}, - "raw": [], - } - - this_data = parameter_data[name] - summary = summary_iterator(str(file)) - for event in summary: - for value in event.summary.value: - if value.tag not in this_data: - this_data[value.tag] = {} - - this_data[value.tag][int(event.step)] = value.simple_value - - if value.tag == SESSION_START_INFO_TAG: - ssi = HParamsPluginData() - ssi.ParseFromString(value.metadata.plugin_data.content) - hparams = dict(ssi.session_start_info.hparams) - for k in hparams: - hparams[k] = hparams[k].ListFields()[0][1] - this_data["hyper_parameters"] = hparams - - break - - json_filename = f"{tensorboard_dir.parent.name}_data.json" - if destination is None: - file_path = tensorboard_dir.parent.joinpath(json_filename) - else: - if not destination.is_dir(): - destination.mkdir() - file_path = destination.joinpath(json_filename) - - with open(file_path, "w") as file: - file.write(json.dumps(parameter_data)) - - return file_path - - -def list_failed(runtime_dir): - runtime_dir = Path(runtime_dir) - failed_list = [] - - for filename in os.listdir(runtime_dir): - with open(runtime_dir.joinpath(str(filename)), "r") as file: - file_lines = file.readlines() - return_code = int(file_lines[-1]) - - if return_code == 0: - continue - - command = file_lines[-3].split("::")[-1].strip() - failed_list.append((filename, command)) - - if len(failed_list) > 0: - for filename, command in failed_list: - # print(f"Failed {runtime_dir} :: {filename} :: {command}") - print(f"!!! {runtime_dir}") - - -def data_from_models(models_dir: Path, destination: Path = None): - if destination is None: - destination = models_dir - - # if not destination.is_dir(): - # destination.mkdir() - if not models_dir.exists(): - print(f"!!! {models_dir}") - return - - for run in next(os.walk(models_dir))[1]: - data = { - "str_weight_model": None, - "str_nn_model": None, - - "parameters_json": None, - "parameter_log": None, - "loss_accuracy": None, - - "hyperparameters_weight_model": None, - "hyperparameters_nn_model": None, - } - run_dir = models_dir.joinpath(str(run)) - run_files = sorted(os.listdir(run_dir), reverse=True) - - looking_for = list(data.keys()) - for i in run_files: - if len(looking_for) == 0: - break - - for j in looking_for: - if j in str(i): - data[j] = i - looking_for.remove(j) - break - - if len(looking_for) > 0: - print(f"Not Found {looking_for}") - print(f"!!! {models_dir}") - continue - - for key, value in data.items(): - data[key] = torch.load(run_dir.joinpath(value)) - - if any([value is None for value in data.values()]): - continue - - with open(destination.joinpath(f"{run}.json"), "w") as file: - file.write(json.dumps(data)) - - -def parse_data(input_path, output_path=None, multiple=False): - input_path = Path(input_path) - output_path = output_path or input_path.joinpath("json") - output_path = Path(output_path) - - if not input_path.is_dir(): - raise Exception(f'"{input_path}" is not a directory or does not exists.') - - if multiple: - all_input_path = [input_path.joinpath(x) for x in (next(os.walk(input_path))[1])] - else: - all_input_path = [input_path] - - for dir_path in all_input_path: - print(f"Processing {dir_path}") - - if not dir_path.joinpath("runtime").exists(): - dir_path = dir_path.joinpath("_results") - - # list_failed(dir_path.joinpath("runtime")) - # data_from_tensorboard(dir_path.joinpath("tensorboard"), output_path) - data_from_models(dir_path.joinpath("models"), output_path) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("--source", type=str, required=True) - parser.add_argument("--output", type=str, default=None) - parser.add_argument("--multiple", action='store_true') - parser.set_defaults(multiple=False) - all_arguments = parser.parse_known_args() - kwargs = vars(all_arguments[0]) - parse_data(kwargs["source"], kwargs['output'], kwargs['multiple']) diff --git a/research/analysis/__init__.py b/research/analysis/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/research/analysis/create_graphs.py b/research/analysis/create_graphs.py deleted file mode 100644 index 168b230..0000000 --- a/research/analysis/create_graphs.py +++ /dev/null @@ -1,805 +0,0 @@ -import itertools -import json -import math -import os -from pathlib import Path -from typing import Tuple - -import matplotlib -import matplotlib.colors -import numpy as np -import seaborn -from matplotlib import pyplot as plt -from seaborn.palettes import _color_to_rgb, _ColorPalette - -from analogvnn.nn.noise.GaussianNoise import GaussianNoise - - -def main_color_palette(n_colors=6, as_cmap=False): # noqa - if as_cmap: - n_colors = 256 - - hues = np.linspace(130, -115, int(n_colors)) % 360 - saturation = np.linspace(1, 1, int(n_colors)) * 99 - lightness = np.linspace(0.85, 0.3, int(n_colors)) * 99 - - palette = [ - _color_to_rgb((h_i, s_i, l_i), input="husl") - for h_i, s_i, l_i in zip(hues, saturation, lightness) - ] - palette = list(reversed(palette)) - if as_cmap: - return matplotlib.colors.ListedColormap(palette, "hsl") - else: - return _ColorPalette(palette) - - -def to_title_case(string: str): - string = string.split(".")[-1] - string = [(x[0].upper() + x[1:].lower()) for x in string.split("_")] - string = " ".join(string) - if string.split(" ")[0] == "Std": - string = " ".join(["σ", *string.split(" ")[1:]]) - string = string.replace(" W", " [W]").replace(" Y", " [Y]") - return string.replace('Leakage', 'Error Probability') - - -def apply_if_not_none(value, func): - if value is None: - return value - return func(value) - - -def sanitise_data(data): - data["train_loss"] = data["loss_accuracy"]["train_loss"][-1] * 100 - data["train_accuracy"] = data["loss_accuracy"]["train_accuracy"][-1] * 100 - data["test_loss"] = data["loss_accuracy"]["test_loss"][-1] * 100 - data["test_accuracy"] = data["loss_accuracy"]["test_accuracy"][-1] * 100 - - data["###"] = int(f"{abs(hash(data['parameter_log']['dataset']))}"[:2]) / 100 - - py = data["hyperparameters_nn_model"]["precision_y"] - pw = data["hyperparameters_weight_model"]["precision_w"] - data["bit_precision_y"] = 32.0 if py is None else math.log2(py) - data["bit_precision_w"] = 32.0 if pw is None else math.log2(pw) - - if "num_linear_layer" in data["parameter_log"]: - data["parameter_log"]["num_linear_layer"] += 1 - if "num_linear_layer" in data["hyperparameters_nn_model"]: - data["hyperparameters_nn_model"]["num_linear_layer"] += 1 - - if data["parameter_log"]["precision_class_w"] == 'None': - data["parameter_log"]["precision_class_w"] = "Digital" - if data["parameter_log"]["precision_class_y"] == 'None': - data["parameter_log"]["precision_class_y"] = "Digital" - - if data["parameter_log"]["precision_y"] is not None \ - and data["parameter_log"]["leakage_y"] is not None: - data["std_y"] = GaussianNoise.calc_std( - data["parameter_log"]["leakage_y"], - data["parameter_log"]["precision_y"] - ) - - if data["parameter_log"]["precision_w"] is not None \ - and data["parameter_log"]["leakage_w"] is not None: - data["std_w"] = GaussianNoise.calc_std( - data["parameter_log"]["leakage_w"], - data["parameter_log"]["precision_w"] - ) - - return data - - -def get_combined_data(data_path): - data_path = Path(data_path) - if data_path.is_file(): - with open(data_path, "r") as file: - data = json.loads(file.read()) - return data - - data = {} - - for i in os.listdir(data_path): - with open(data_path.joinpath(str(i)), "r") as file: - dd = json.loads(file.read()) - data[str(i)] = sanitise_data(dd) - with open(data_path.joinpath(str(i)), "w") as file: - file.write(json.dumps(dd, indent=2, sort_keys=True)) - - return data - - -def compile_data(data_path): - data_path = Path(data_path) - run_data = get_combined_data(data_path) - with open(f"{data_path}.json", "w") as file: - file.write(json.dumps(run_data, indent=2)) - - -def get_key(obj, key): - key = key.split(".") - for i in key: - obj = obj[i] - return obj - - -def get_filtered_data(data, filters): - if filters is None or len(filters.keys()) == 0: - return data - - filtered_data = {} - - for key, value in data.items(): - check = True - for filter_key, filter_value in filters.items(): - if isinstance(filter_value, str): - if not any([get_key(data[key], filter_key) == i for i in filter_value.split("|")]): - check = False - break - else: - if get_key(data[key], filter_key) != filter_value: - check = False - break - - if check: - filtered_data[key] = value - - return filtered_data - - -def get_plot_data(data_path, x_axis, y_axis, subsection=None, colorbar=None, filters=None, add_data=None): - if add_data is None: - add_data = {} - - plot_labels = {} - plot_data = {} - add_data["x"] = x_axis - add_data["y"] = y_axis - add_data["hue"] = subsection - add_data["style"] = colorbar - - if isinstance(data_path, list): - run_data = {} - for i in data_path: - data = get_combined_data(Path(i)) - run_data = {**run_data, **data} - else: - run_data = get_combined_data(Path(data_path)) - - run_data = get_filtered_data(run_data, filters) - - for key, value in add_data.items(): - if value is None: - continue - - plot_labels[key] = value - plot_data[key] = [] - - for key, value in run_data.items(): - for i in plot_labels: - plot_data[i].append(get_key(run_data[key], plot_labels[i])) - - if colorbar is None: - if subsection is not None: - plot_data["hue_order"] = sorted(list(set(plot_data["hue"]))) - if "Digital" in plot_data["hue_order"]: - plot_data["hue_order"].remove("Digital") - plot_data["hue_order"].insert(0, "Digital") - if "None" in plot_data["hue_order"]: - plot_data["hue_order"].remove("None") - plot_data["hue_order"].insert(0, "None") - else: - if "hue" not in plot_data: - plot_data["hue"] = plot_data["style"] - del plot_data["style"] - else: - plot_data["hue"], plot_data["style"] = plot_data["style"], plot_data["hue"] - - zip_list = ["x", "y"] - if "hue" in plot_data: - zip_list.append("hue") - if "style" in plot_data: - zip_list.append("style") - - if isinstance(plot_data["x"][0], str): - ziped_list = list(zip(*[plot_data[x] for x in zip_list])) - ziped_list = sorted(ziped_list, key=lambda tup: -np.sum(np.array(tup[0]) == "None")) - unziped_list = list(zip(*ziped_list)) - - for i, v in enumerate(zip_list): - plot_data[v] = list(unziped_list[i]) - return plot_data - - -def pick_max_from_plot_data(plot_data, max_keys, max_value): - max_keys_value = [] - for i in max_keys: - max_keys_value.append(list(set(plot_data[i]))) - - max_accuracies = {i: 0 for i in list(itertools.product(*max_keys_value))} - for index, value in enumerate(plot_data[max_value]): - index_max_key_values = tuple([plot_data[i][index] for i in max_keys]) - - if max_accuracies[index_max_key_values] < value: - max_accuracies[index_max_key_values] = value - - plot_data[max_value] = [] - for i in max_keys: - plot_data[i] = [] - - max_accuracies = sorted(max_accuracies.items(), key=lambda tup: tup[0]) - max_accuracies = sorted(max_accuracies, key=lambda tup: -np.sum(np.array(tup[0]) == "None")) - - for key, value in max_accuracies: - for index, val in enumerate(max_keys): - plot_data[max_keys[index]].append(key[index]) - - plot_data[max_value].append(value) - return plot_data - - -def pre_plot(size_factor): - matplotlib.rcParams['pdf.fonttype'] = 42 - matplotlib.rcParams['ps.fonttype'] = 42 - - fig = plt.figure() - ax = fig.add_subplot(111) - for axis in ['top', 'bottom', 'left', 'right']: - ax.spines[axis].set_linewidth(0.75) - - # fig_size = [3.25, 1.85] - # fig_size = [2.00, 1.75] - fig_size = 1.75 - - fig_size = tuple((np.array(fig_size) * np.array(size_factor)).tolist()) - fig.set_size_inches(*fig_size) - fig.set_dpi(200) - - return fig - - -def post_plot(plot_data): - x_axis_title = to_title_case(plot_data["x_axis"]) - y_axis_title = to_title_case(plot_data["y_axis"]) - filter_text = "" - - if plot_data["filters"] is not None: - filter_text = " {" + "-".join( - [f"{to_title_case(key)}={value}" for key, value in plot_data["filters"].items()]) + "}" - filter_text = filter_text.replace("|", " or ") - # plt.title(f"Filters = {filter_text}") - - plt.yticks(np.arange(0, 101, 25)) - plt.ylim([0, 100]) - plt.xlabel(x_axis_title) - # plt.ylabel(y_axis_title) - plt.ylabel((plot_data["y_prefix"] if "y_prefix" in plot_data else "") + y_axis_title) - - if plot_data["subsection"] is not None: - if "g" in plot_data: - h, l = plot_data["g"].get_legend_handles_labels() - - if plot_data["colorbar"] is None: - subsection_len = len(set(plot_data["hue"])) - else: - subsection_len = len(set(plot_data["style"])) - - plt.legend(h[-subsection_len:], l[-subsection_len:], title=to_title_case(plot_data["subsection"])) - else: - plt.legend(title=to_title_case(plot_data["subsection"])) - elif plot_data["colorbar"] is not None: - plt.legend(title=to_title_case(plot_data["colorbar"])) - - plot_data["fig"].tight_layout() - # plt.show() - - if isinstance(plot_data["data_path"], list): - run_name = "".join([Path(i).name for i in plot_data["data_path"]]) - else: - run_name = Path(plot_data["data_path"]).name[:Path(plot_data["data_path"]).name.index(".")] - - subsection_text = "" if plot_data["subsection"] is None else f" #{to_title_case(plot_data['subsection'])}" - colorbar_text = "" if plot_data["colorbar"] is None else f" #{to_title_case(plot_data['colorbar'])}" - - name = f"{plot_data['prefix']} - {run_name} - {x_axis_title} vs {y_axis_title}{filter_text}{colorbar_text}{subsection_text}" - - plot_data["fig"].savefig(f'{location}/{name}.svg', dpi=plot_data["fig"].dpi, transparent=True) - plot_data["fig"].savefig(f'{location}/{name}.png', dpi=plot_data["fig"].dpi, transparent=True) - - plt.close('all') - - -def create_violin_figure(data_path, x_axis, y_axis, subsection=None, colorbar=None, filters=None, - size_factor: Tuple[float, float] = 2, color_by=None): - if filters is None: - filters = {} - if colorbar is not None: - subsection = colorbar - - plot_data = get_plot_data(data_path, x_axis, y_axis, subsection=subsection, filters=filters, - add_data={"color_by": color_by}) - - fig = pre_plot(size_factor) - # color_by_data = None - # if color_by is not None: - # color_by_data = plot_data["color_by"] - # del plot_data["color_by"] - - n_colors = None - n_colors = len(plot_data["hue_order"]) if ("hue" in plot_data and n_colors is None) else n_colors - n_colors = len(set(plot_data["x"])) if n_colors is None else n_colors - color_map = main_color_palette(n_colors=n_colors) - g = seaborn.violinplot(**plot_data, cut=0, palette=color_map, inner=None, linewidth=0.1) - color_map = main_color_palette(n_colors=n_colors) - - # if color_by is not None: - # for i, color_by_value in enumerate(set(color_by_data)): - # new_plot_data = get_plot_data(data_path, x_axis, y_axis, subsection=subsection, filters={ - # **filters, - # color_by: color_by_value - # }) - # gs = seaborn.stripplot(**new_plot_data, palette=[color_map[len(set(plot_data["x"])) + i]], linewidth=0.1, size=3, jitter=1 / 10, dodge=True) - # else: - # gs = seaborn.stripplot(**plot_data, palette=color_map, linewidth=0.1, size=3, jitter=1 / 10, dodge=True) - gs = seaborn.stripplot(**plot_data, palette=color_map, linewidth=0.1, size=3, jitter=1 / 10, dodge=True) - if colorbar is not None: - color_map = matplotlib.colors.LinearSegmentedColormap.from_list("hsl", color_map) - # color_map = seaborn.dark_palette("#69d", n_colors=len(set(plot_data["hue"])), reverse=True, as_cmap=True) - norm = matplotlib.colors.Normalize(vmin=min(plot_data["hue"]), vmax=max(plot_data["hue"])) - scalar_map = plt.cm.ScalarMappable(cmap=color_map, norm=norm) - cbar = plt.colorbar(scalar_map) - cbar.ax.set_ylabel(to_title_case(colorbar)) - # if color_by is not None: - # color_by_data, plot_data["hue"] = plot_data["hue"], color_by_data - # plot_data["hue_order"] = sorted(list(set(plot_data["hue"]))) - - # if color_by is not None: - # color_map = seaborn.color_palette("flare", len(set(plot_data["hue"])), as_cmap=True) - # # color_map = seaborn.dark_palette("#69d", n_colors=len(set(plot_data["hue"])), reverse=True, as_cmap=True) - # norm = matplotlib.colors.Normalize(vmin=min(plot_data["hue"]), vmax=max(plot_data["hue"])) - # scalar_map = plt.cm.ScalarMappable(cmap=color_map, norm=norm) - # scalar_map.set_array([]) - # cbar = fig.colorbar(scalar_map, ax=gs) - # cbar.ax.set_title("\"bins\"") - - # if color_by is not None: - # color_by_data, plot_data["hue"] = plot_data["hue"], color_by_data - # plot_data["hue_order"] = sorted(list(set(plot_data["hue"]))) - - plot_data["data_path"] = data_path - plot_data["prefix"] = "v" - plot_data["fig"] = fig - plot_data["g"] = g - plot_data["gs"] = gs - plot_data["x_axis"] = x_axis - plot_data["y_axis"] = y_axis - plot_data["subsection"] = subsection - plot_data["colorbar"] = None - plot_data["filters"] = filters - post_plot(plot_data) - - -def create_line_figure(data_path, x_axis, y_axis, subsection=None, colorbar=None, filters=None, - size_factor: Tuple[float, float] = 2, ci=1): - plot_data = get_plot_data(data_path, x_axis, y_axis, subsection=subsection, colorbar=colorbar, filters=filters) - - fig = pre_plot(size_factor) - - color_map = main_color_palette(n_colors=10) - if colorbar is not None: - color_map = matplotlib.colors.LinearSegmentedColormap.from_list("hsl", color_map) - # color_map = seaborn.dark_palette("#69d", n_colors=len(set(plot_data["hue"])), reverse=True, as_cmap=True) - norm = matplotlib.colors.Normalize(vmin=min(plot_data["hue"]), vmax=max(plot_data["hue"])) - scalar_map = plt.cm.ScalarMappable(cmap=color_map, norm=norm) - cbar = plt.colorbar(scalar_map) - cbar.ax.set_ylabel(to_title_case(colorbar)) - - g = seaborn.lineplot(**plot_data, palette=color_map, linewidth=1, ci=ci) - - plot_data["data_path"] = data_path - plot_data["prefix"] = "l" - plot_data["fig"] = fig - plot_data["g"] = g - plot_data["x_axis"] = x_axis - plot_data["y_axis"] = y_axis - plot_data["subsection"] = subsection - plot_data["colorbar"] = colorbar - plot_data["filters"] = filters - plot_data["y_prefix"] = "Average " - post_plot(plot_data) - - -def create_line_figure_max(data_path, x_axis, y_axis, subsection=None, colorbar=None, filters=None, - size_factor: Tuple[float, float] = 2.0, x_lim=None): - plot_data = get_plot_data(data_path, x_axis, y_axis, subsection=subsection, colorbar=colorbar, filters=filters) - fig = pre_plot(size_factor) - - max_keys = ["x"] - - if subsection is not None: - max_keys.append("hue") - if colorbar is not None and subsection is not None: - max_keys.append("style") - if colorbar is not None and subsection is None: - max_keys.append("hue") - - plot_data = pick_max_from_plot_data(plot_data, max_keys, "y") - - if colorbar is not None: - color_map = main_color_palette(n_colors=256) - color_map = matplotlib.colors.LinearSegmentedColormap.from_list("hsl", color_map) - # color_map = seaborn.color_palette("cubehelix", len(set(plot_data["hue"])), as_cmap=True) - # color_map = seaborn.dark_palette("#69d", n_colors=len(set(plot_data["hue"])), reverse=True, as_cmap=True) - norm = matplotlib.colors.Normalize(vmin=min(plot_data["hue"]), vmax=0.2) - scalar_map = plt.cm.ScalarMappable(cmap=color_map, norm=norm) - cbar = plt.colorbar(scalar_map) - cbar.ax.set_ylabel(to_title_case(colorbar)) - g = seaborn.lineplot(**plot_data, palette=color_map, linewidth=1, errorbar=('ci', 1), hue_norm=norm) - else: - color_map = main_color_palette(n_colors=len(plot_data["hue_order"])) - g = seaborn.lineplot(**plot_data, palette=color_map, linewidth=1, errorbar=('ci', 1)) - - if x_lim is not None: - g.set_xlim(*x_lim) - # g.set(yscale="log") - - plot_data["data_path"] = data_path - plot_data["prefix"] = "lm" - plot_data["fig"] = fig - plot_data["g"] = g - plot_data["x_axis"] = x_axis - plot_data["y_axis"] = y_axis - plot_data["subsection"] = subsection - plot_data["colorbar"] = colorbar - plot_data["filters"] = filters - plot_data["y_prefix"] = "Maximum " - post_plot(plot_data) - - -def calculate_max_accuracy(data_path, test_in): - data_path = Path(data_path) - plot_data = get_plot_data(data_path, test_in, "loss_accuracy.test_accuracy") - max_accuracies = {} - for i in set(plot_data["x"]): - max_accuracies[i] = 0.0 - - for index, value in enumerate(plot_data["y"]): - value = max(value) - if max_accuracies[plot_data["x"][index]] < value: - max_accuracies[plot_data["x"][index]] = value - max_accuracies[plot_data["x"][index] + "_index"] = index - - print(max_accuracies) - - -def create_analogvnn1_figures_va1(): - create_line_figure_max( - f"{location}/analogvnn_1.2_json.json", - "parameter_log.num_linear_layer", - "test_accuracy", - colorbar="parameter_log.num_conv_layer", - subsection="parameter_log.dataset", - size_factor=(6.5 * 1 / 3, 1.61803398874), - filters={ - "parameter_log.norm_class_w": "None", - "parameter_log.norm_class_y": "None", - } - ) - create_violin_figure( - f"{location}/analog_vnn_3.json", - "parameter_log.activation_class", - "test_accuracy", - size_factor=(6.5 * 2 / 3, 1.61803398874), - subsection="parameter_log.dataset", - ) - - -def create_analogvnn1_figures_va2(): - create_line_figure_max( - f"{location}/analogvnn_1.2_json.json", - "parameter_log.norm_class_w", - "test_accuracy", - subsection="parameter_log.norm_class_y", - size_factor=(6.5, 2), - ) - create_violin_figure( - f"{location}/analogvnn_1.2_json.json", - "parameter_log.norm_class_w", - "test_accuracy", - size_factor=(6.5, 2), - subsection="parameter_log.dataset", - ) - - -def create_analogvnn1_figures_va3(): - create_violin_figure( - f"{location}/analog_vnn_2.json", - "bit_precision_w", - "test_accuracy", - subsection="parameter_log.precision_class_w", - size_factor=(6.5 * 1 / 3, 1.61803398874), - color_by="bit_precision_w", - filters={ - "parameter_log.dataset": "MNIST", - "parameter_log.norm_class_w": "Clamp", - "parameter_log.norm_class_y": "Clamp", - }, - ) - create_violin_figure( - f"{location}/analog_vnn_2.json", - "bit_precision_w", - "test_accuracy", - subsection="parameter_log.precision_class_w", - size_factor=(6.5 * 1 / 3, 1.61803398874), - color_by="bit_precision_w", - filters={ - "parameter_log.dataset": "FashionMNIST", - "parameter_log.norm_class_w": "Clamp", - "parameter_log.norm_class_y": "Clamp", - }, - ) - create_violin_figure( - f"{location}/analog_vnn_2.json", - "bit_precision_w", - "test_accuracy", - subsection="parameter_log.precision_class_w", - size_factor=(6.5 * 1 / 3, 1.61803398874), - color_by="bit_precision_w", - filters={ - "parameter_log.dataset": "CIFAR10", - "parameter_log.norm_class_w": "Clamp", - "parameter_log.norm_class_y": "Clamp", - }, - ) - create_line_figure( - f"{location}/analog_vnn_3.json", - "bit_precision_w", - "test_accuracy", - colorbar="bit_precision_y", - subsection="parameter_log.dataset", - size_factor=(6.5 * 1 / 2, 1.61803398874 * 1.2), - ) - create_line_figure_max( - f"{location}/analog_vnn_3.json", - "bit_precision_w", - "test_accuracy", - colorbar="bit_precision_y", - subsection="parameter_log.dataset", - size_factor=(6.5 * 1 / 2, 1.61803398874 * 1.2), - ) - - -def create_analogvnn1_figures_va4(): - create_violin_figure( - f"{location}/analog_vnn_3.json", - "parameter_log.dataset", - "test_accuracy", - subsection="parameter_log.leakage_w", - size_factor=(6.5 * 1 / 3, 1.61803398874), - ) - create_line_figure( - f"{location}/analog_vnn_3.json", - "parameter_log.leakage_w", - "test_accuracy", - colorbar="parameter_log.leakage_y", - subsection="parameter_log.dataset", - size_factor=(6.5 * 1 / 3, 1.61803398874), - ) - create_line_figure_max( - f"{location}/analog_vnn_3.json", - "parameter_log.leakage_w", - "test_accuracy", - colorbar="parameter_log.leakage_y", - subsection="parameter_log.dataset", - size_factor=(6.5 * 1 / 3, 1.61803398874), - ) - create_line_figure_max( - f"{location}/analog_vnn_3.json", - "parameter_log.leakage_w", - "test_accuracy", - colorbar="bit_precision_w", - subsection="parameter_log.dataset", - size_factor=(6.5 * 1 / 2, 1.61803398874 * 1.2), - ) - create_line_figure_max( - f"{location}/analog_vnn_3.json", - "std_w", - "test_accuracy", - colorbar="std_y", - subsection="parameter_log.dataset", - size_factor=(6.5 * 1 / 2, 1.61803398874 * 1.2), - ) - - -def create_parneet_figures_vb1(): - create_line_figure( - f"{location}/runs_parneet_1_b_json.json", - "parameter_log.batch_size", - "test_accuracy", - ci="sd", - size_factor=(6.5 * 1 / 2, 1.61803398874 * 1.2), - ) - create_line_figure( - f"{location}/runs_parneet_1_b_json.json", - "parameter_log.activation_class", - "test_accuracy", - colorbar="parameter_log.batch_size", - size_factor=(6.5 * 1 / 2, 1.61803398874 * 1.2), - ) - create_line_figure_max( - f"{location}/runs_parneet_2_n3_json.json", - "parameter_log.norm_class_w", - "test_accuracy", - subsection="parameter_log.norm_class_y", - size_factor=(6.5, 1.61803398874), - ) - # create_violin_figure( - # f"{location}/runs_parneet_2_n3_json.json", - # "parameter_log.norm_class_w", - # "test_accuracy", - # size_factor=(6.5, 1.61803398874 * 1.2), - # ) - - -def create_parneet_figures_vb2(): - create_line_figure_max( - f"{location}/runs_parneet_3_p_json.json", - "bit_precision_w", - "test_accuracy", - colorbar="bit_precision_y", - size_factor=(6.5 / 2, 1.61803398874 * 1.2), - ) - create_line_figure_max( - f"{location}/runs_parneet_3_p_json.json", - "bit_precision_w", - "test_accuracy", - subsection="parameter_log.activation_class", - size_factor=(6.5 / 2, 1.61803398874 * 1.2), - ) - - -def create_parneet_figures_vb3(): - # create_violin_figure( - # f"{location}/runs_parneet_4_g_json.json", - # "parameter_log.color", - # "test_accuracy", - # colorbar="bit_precision_w", - # size_factor=(6.5 * 1/3, 1.61803398874), - # ) - # create_line_figure_max( - # f"{location}/runs_parneet_4_g_json.json", - # "parameter_log.leakage_w", - # "test_accuracy", - # subsection="parameter_log.activation_class", - # size_factor=(6.5 * 1/3, 1.61803398874), - # ) - # create_line_figure_max( - # f"{location}/runs_parneet_4_g_json.json", - # "bit_precision_w", - # "test_accuracy", - # colorbar="bit_precision_y", - # size_factor=(6.5 * 1/3, 1.61803398874), - # ) - - create_line_figure_max( - f"{location}/runs_parneet_4_g_json.json", - "parameter_log.leakage_w", - "test_accuracy", - colorbar="parameter_log.leakage_y", - size_factor=(6.5 * 1 / 3, 1.61803398874), - ) - # create_line_figure_max( - # f"{location}/runs_parneet_4_g_json.json", - # "parameter_log.leakage_w", - # "test_accuracy", - # colorbar="bit_precision_w", - # size_factor=(6.5 * 1/3, 1.61803398874), - # ) - # create_line_figure_max( - # f"{location}/runs_parneet_4_g_json.json", - # "std_w", - # "test_accuracy", - # colorbar="std_y", - # x_lim=[0, 0.08], - # size_factor=(6.5 * 1/3, 1.61803398874), - # ) - - -def create_analogvnn1_figures(): - create_analogvnn1_figures_va1() - create_analogvnn1_figures_va2() - # create_analogvnn1_figures_va3() - create_analogvnn1_figures_va4() - - -def create_parneet_figures(): - create_parneet_figures_vb1() - # create_parneet_figures_vb2() - # create_parneet_figures_vb3() - - -if __name__ == '__main__': - location = "C:/_data" - # compile_data(f"{location}/analog_vnn_1") - # compile_data(f"{location}/analog_vnn_2") - # compile_data(f"{location}/analog_vnn_3") - # compile_data(f"{location}/analogvnn_1.2_json") - - # compile_data(f"{location}/runs_parneet_1_b_json") - # compile_data(f"{location}/runs_parneet_2_n3_json") - # compile_data(f"{location}/runs_parneet_2_n_json") - # compile_data(f"{location}/runs_parneet_3_p_json") - # compile_data(f"{location}/runs_parneet_4_g_json") - - # create_analogvnn1_figures() - # create_parneet_figures() - - create_line_figure_max( - f"{location}/runs_parneet_4_g_json.json", - "std_w", - "test_accuracy", - colorbar="std_y", - x_lim=[0, 0.08], - size_factor=(6.5 * 1.25 / 3, 1.61803398874), - ) - - # data = get_combined_data(Path(f"{location}/analogvnn_1.2_json.json")) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.MNIST.__name__, "parameter_log.num_linear_layer":1, "parameter_log.num_conv_layer":0}) - # print("MNIST 0,1 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.MNIST.__name__, "parameter_log.num_linear_layer":2, "parameter_log.num_conv_layer":0}) - # print("MNIST 0,2 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.MNIST.__name__, "parameter_log.num_linear_layer":3, "parameter_log.num_conv_layer":0}) - # print("MNIST 0,3 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.MNIST.__name__, "parameter_log.num_linear_layer":1, "parameter_log.num_conv_layer":3}) - # print("MNIST 3,1 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.MNIST.__name__, "parameter_log.num_linear_layer":2, "parameter_log.num_conv_layer":3}) - # print("MNIST 3,2 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.MNIST.__name__, "parameter_log.num_linear_layer":3, "parameter_log.num_conv_layer":3}) - # print("MNIST 3,3 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.FashionMNIST.__name__, "parameter_log.num_linear_layer":1, "parameter_log.num_conv_layer":0}) - # print("FashionMNIST 0,1 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.FashionMNIST.__name__, "parameter_log.num_linear_layer":2, "parameter_log.num_conv_layer":0}) - # print("FashionMNIST 0,2 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.FashionMNIST.__name__, "parameter_log.num_linear_layer":3, "parameter_log.num_conv_layer":0}) - # print("FashionMNIST 0,3 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.FashionMNIST.__name__, "parameter_log.num_linear_layer":1, "parameter_log.num_conv_layer":3}) - # print("FashionMNIST 3,1 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.FashionMNIST.__name__, "parameter_log.num_linear_layer":2, "parameter_log.num_conv_layer":3}) - # print("FashionMNIST 3,2 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.FashionMNIST.__name__, "parameter_log.num_linear_layer":3, "parameter_log.num_conv_layer":3}) - # print("FashionMNIST 3,3 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.CIFAR10.__name__, "parameter_log.num_linear_layer":1, "parameter_log.num_conv_layer":0}) - # print("CIFAR10 0,1 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.CIFAR10.__name__, "parameter_log.num_linear_layer":2, "parameter_log.num_conv_layer":0}) - # print("CIFAR10 0,2 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.CIFAR10.__name__, "parameter_log.num_linear_layer":3, "parameter_log.num_conv_layer":0}) - # print("CIFAR10 0,3 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.CIFAR10.__name__, "parameter_log.num_linear_layer":1, "parameter_log.num_conv_layer":3}) - # print("CIFAR10 3,1 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.CIFAR10.__name__, "parameter_log.num_linear_layer":2, "parameter_log.num_conv_layer":3}) - # print("CIFAR10 3,2 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.CIFAR10.__name__, "parameter_log.num_linear_layer":3, "parameter_log.num_conv_layer":3}) - # print("CIFAR10 3,3 max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - - # data = get_combined_data(Path(f"{location}/analog_vnn_3.json")) - # filtered_data = get_filtered_data(data, {"parameter_log.dataset": torchvision.datasets.MNIST.__name__, - # "bit_precision_w": 4, "bit_precision_y": 2}) - # print("MNIST max: ", np.max([get_key(data[key], "test_accuracy") for key, value in filtered_data.items()])) - - # create_line_figure_max( - # f"{location}/runs_parneet_4_g_json.json", - # "std_w", - # "test_accuracy", - # colorbar="std_y", - # size_factor=(3.25, 1.5), - # ) - # colormaps = ["rocket", "mako", "flare", "crest", "magma", "viridis", "rocket_r", "cubehelix", "seagreen", "dark:salmon_r", "YlOrBr", "Blues", "vlag", "icefire", "coolwarm", "Spectral"] - # colormaps_dict = {} - # for i in colormaps: - # try: - # colormaps_dict["cm_" + i] = seaborn.color_palette(i, n_colors=256, as_cmap=True).colors - # except Exception as e: - # print(i) - # scipy.io.savemat("seaborn.colormap.mat", colormaps_dict) diff --git a/research/analysis/graph_gaussian_noise.py b/research/analysis/graph_gaussian_noise.py deleted file mode 100644 index 9f21e6a..0000000 --- a/research/analysis/graph_gaussian_noise.py +++ /dev/null @@ -1,515 +0,0 @@ -import csv -import json -import math -from dataclasses import dataclass -from pathlib import Path -from typing import List, Callable - -import numpy as np -import scipy.io -import torch -from matplotlib import pyplot as plt -from scipy import integrate -from scipy.io import savemat - -from analogvnn.fn.dirac_delta import dirac_delta -from analogvnn.nn.module.Sequential import Sequential -from analogvnn.nn.noise.GaussianNoise import GaussianNoise -from analogvnn.nn.noise.LaplacianNoise import LaplacianNoise -from analogvnn.nn.noise.PoissonNoise import PoissonNoise -from analogvnn.nn.noise.UniformNoise import UniformNoise -from analogvnn.nn.precision.ReducePrecision import ReducePrecision -from analogvnn.utils.is_cpu_cuda import is_cpu_cuda - - -def calculate_leakage(x, rp_normal_noise): - return torch.sum(torch.sign(torch.abs(x - rp_normal_noise))) / np.prod(x.size()) - - -def calculate_weighted_leakage(precision, x, rp_normal_noise): - return torch.sum(torch.abs(x - rp_normal_noise)) * precision / np.prod(x.size()) - - -def calculate_snr(signal, noise_signal): - t2 = torch.tensor(2, device=signal.device) - s = torch.sum(torch.pow(signal, t2)) - n = torch.sum(torch.pow(signal - noise_signal, t2)) - return s / n - - -@dataclass -class CalculateResponds: - signal: torch.Tensor - rp: torch.Tensor - noise_rp: torch.Tensor - rp_noise_rp: torch.Tensor - - leakage: torch.Tensor - weighted_leakage: torch.Tensor - digital_analog_snr: torch.Tensor - analog_analog_snr: torch.Tensor - digital_digital_snr: torch.Tensor - - func: Callable - - -class CalculateRespondsAverages: - def __init__(self): - self.leakage: List[torch.Tensor] = [] - self.weighted_leakage: List[torch.Tensor] = [] - self.digital_analog_snr: List[torch.Tensor] = [] - self.analog_analog_snr: List[torch.Tensor] = [] - self.digital_digital_snr: List[torch.Tensor] = [] - self._values = None - - def append_response(self, response: CalculateResponds): - self.leakage.append(response.leakage) - self.weighted_leakage.append(response.weighted_leakage) - self.digital_analog_snr.append(response.digital_analog_snr) - self.analog_analog_snr.append(response.analog_analog_snr) - self.digital_digital_snr.append(response.digital_digital_snr) - self._values = None - - @property - def values(self): - if self._values is None: - self._values = ( - sum(self.leakage) / len(self.leakage), - sum(self.weighted_leakage) / len(self.weighted_leakage), - sum(self.digital_analog_snr) / len(self.digital_analog_snr), - sum(self.analog_analog_snr) / len(self.analog_analog_snr), - sum(self.digital_digital_snr) / len(self.digital_digital_snr) - ) - return self._values - - def __repr__(self): - return "leakage: {0:.4f}" \ - ", weighted_leakage: {1:.4f}" \ - ", digital_analog_snr: {2:.4f}" \ - ", analog_analog_snr: {3:.4f}" \ - ", digital_digital_snr: {4:.4f}" \ - "".format(*self.values) - - -def calculate(signal, precision, noise_fn) -> CalculateResponds: - rp_signal = ReducePrecision(precision=precision)(signal) - noise_rp_signal = noise_fn(rp_signal) - rp_noise_rp_signal = ReducePrecision(precision=precision)(noise_rp_signal) - - return CalculateResponds( - signal=signal, - rp=rp_signal, - noise_rp=noise_rp_signal, - rp_noise_rp=rp_noise_rp_signal, - - leakage=calculate_leakage(rp_signal, rp_noise_rp_signal), - weighted_leakage=calculate_weighted_leakage(precision, rp_signal, rp_noise_rp_signal), - digital_analog_snr=calculate_snr(signal, noise_rp_signal), - analog_analog_snr=calculate_snr(rp_signal, noise_rp_signal), - digital_digital_snr=calculate_snr(signal, rp_noise_rp_signal), - func=noise_fn, - ) - - -def create_normal_snr(): - folder_path = Path("C:/_data") - is_cpu_cuda.set_device('cpu') - leakages = np.linspace(0, 1, 101) - precisions = np.array(range(1, 2 ** 8)) - - x_values = torch.Tensor(np.linspace(-1, 1, 1000)) - - data = [ - ["precision", "leakage", "calculated_leakages", "weighted_leakage", "analog_snr", "digital_snr"], - ] - result_str = "" - for i, precision in enumerate(precisions): - for j, leakage in enumerate(leakages): - average_response = CalculateRespondsAverages() - for k in range(1000): - average_response.append_response(calculate( - x_values, float(precision), - lambda x: GaussianNoise(leakage=leakage, precision=precision)(x) - )) - data.append([precision, leakage] + [float(x) for x in average_response.values]) - print_str = f"{i}, {j}: {abs(average_response.values[0] - leakage):.4f} == 0, precision: {precision}, leakage: {leakage:.4f}, {average_response}" - print(print_str) - result_str += print_str + "\n" - - with open(folder_path.joinpath("normal_noise.txt"), "w") as file: - file.write(result_str) - with open(folder_path.joinpath("normal_noise.json"), "w") as file: - file.write(json.dumps(data)) - with open(folder_path.joinpath("normal_noise.csv"), 'w', newline='') as file: - csv.writer(file, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL).writerows(data) - - data_shape = (precisions.size, leakages.size) - calculated_leakages = np.zeros(data_shape) - weighted_leakage = np.zeros(data_shape) - analog_snr = np.zeros(data_shape) - digital_snr = np.zeros(data_shape) - - for i, v in enumerate(data[1:]): - precision_index = int(np.where(precisions == v[0])[0]) - leakage_index = int(np.where(leakages == v[1])[0]) - calculated_leakages[precision_index][leakage_index] = v[2] - weighted_leakage[precision_index][leakage_index] = v[3] - analog_snr[precision_index][leakage_index] = v[4] - digital_snr[precision_index][leakage_index] = v[5] - - scipy.io.savemat(folder_path.joinpath("normal_noise.mat"), { - "leakages": leakages, - "precisions": precisions, - "calculated_leakages": calculated_leakages, - "weighted_leakage": weighted_leakage, - "analog_snr": analog_snr, - "digital_snr": digital_snr, - }) - - -def main(std=0.1, precision=3): - is_cpu_cuda.set_device('cpu') - - xlim = [-1.01, 1.01] - ylim = [-1.5, 1.5] - - num_line = torch.Tensor(np.linspace(-1, 1, 500)) - reduce_precision = ReducePrecision(precision=precision)(num_line) - - sp = [3, 3] - fig, axes = plt.subplots(nrows=sp[0], ncols=sp[1]) - # fig.set_size_inches(4.0, 4.0) - fig.set_size_inches(6, 6) - fig.set_dpi(200) - - plot_x = num_line.tolist() - plt.subplot(*sp, 1) - plt.plot(plot_x, plot_x, color="#ff0000") - plt.gca().set_xlim(xlim) - plt.gca().set_ylim(ylim) - plt.subplot(*sp, 3) - plt.plot(plot_x, (np.array(plot_x) * 1.5).tolist(), color="#ff0000") - plt.gca().set_xlim(xlim) - plt.gca().set_ylim(ylim) - - plt.subplot(*sp, 2) - plt.plot(plot_x, reduce_precision.tolist(), label="reduce_precision", color="#ff0000") - plt.gca().set_xlim(xlim) - plt.gca().set_ylim(ylim) - plt.title(f"Reduce Precision (p: {precision} = {math.log2(precision):0.3f} bits)") - - plt.subplot(*sp, 4) - plt.gca().set_xlim(xlim) - plt.gca().set_ylim(ylim) - plt.title(f"Normal") - - plt.subplot(*sp, 5) - plt.gca().set_xlim(xlim) - plt.gca().set_ylim(ylim) - plt.title(f"Possion (scale: {std})") - - plt.subplot(*sp, 6) - plt.gca().set_xlim(xlim) - plt.gca().set_ylim(ylim) - plt.title(f"Normal + Poisson") - - uniform_avg = CalculateRespondsAverages() - normal_avg = CalculateRespondsAverages() - poisson_avg = CalculateRespondsAverages() - poisson_ones_avg = CalculateRespondsAverages() - laplace_avg = CalculateRespondsAverages() - normal_poisson_avg = CalculateRespondsAverages() - - un = UniformNoise(leakage=std, precision=precision) - gn = GaussianNoise(leakage=std, precision=precision) - pn = PoissonNoise(max_leakage=0.1, precision=precision) - ln = LaplacianNoise(scale=1 / std, precision=precision) - for i in range(1000): - uniform_noise = calculate(num_line, precision, un) - normal_noise = calculate(num_line, precision, gn) - poisson_noise = calculate(num_line, precision, pn) - poisson_noise_ones = calculate(torch.ones_like(num_line), precision, pn) - laplace_noise = calculate(num_line, precision, ln) - normal_poisson_noise = calculate( - num_line, precision, - lambda x: normal_noise.func(poisson_noise.func(x)) - ) - - uniform_avg.append_response(uniform_noise) - normal_avg.append_response(normal_noise) - poisson_avg.append_response(poisson_noise) - poisson_ones_avg.append_response(poisson_noise_ones) - laplace_avg.append_response(laplace_noise) - normal_poisson_avg.append_response(normal_poisson_noise) - - if i < 5: - plt.subplot(*sp, 4) - plt.scatter(plot_x, normal_noise.noise_rp.tolist(), color="#ff00000f", s=2) - - plt.subplot(*sp, 5) - plt.scatter(plot_x, poisson_noise.noise_rp.tolist(), color="#ff00000f", s=2) - - plt.subplot(*sp, 6) - plt.scatter(plot_x, normal_poisson_noise.noise_rp.tolist(), color="#ff00000f", s=2) - - plt.subplot(*sp, 7) - plt.scatter(plot_x, normal_noise.rp_noise_rp.tolist(), color="#ff00000f", s=2) - - plt.subplot(*sp, 8) - plt.scatter(plot_x, poisson_noise.rp_noise_rp.tolist(), color="#ff00000f", s=2) - - plt.subplot(*sp, 9) - plt.scatter(plot_x, normal_poisson_noise.rp_noise_rp.tolist(), color="#ff00000f", s=2) - - plt.subplot(*sp, 7) - plt.gca().set_xlim(xlim) - plt.gca().set_ylim(ylim) - plt.subplot(*sp, 8) - plt.gca().set_xlim(xlim) - plt.gca().set_ylim(ylim) - plt.subplot(*sp, 9) - plt.gca().set_xlim(xlim) - plt.gca().set_ylim(ylim) - - print(f"std: {std:0.4f}, precision: {precision}") - print(f"uniform_expected: {un.leakage:0.4f}, {uniform_avg}") - print(f"normal_expected: {gn.leakage:0.4f}, {normal_avg}") - print(f"poisson_expected: {pn.leakage:0.4f}, {poisson_avg}") - print(f"poisson_ones_expected: {pn.max_leakage:0.4f}, {poisson_ones_avg}") - print(f"laplace_expected: {ln.leakage:0.4f}, {laplace_avg}") - print(f"normal_poisson_avg: {normal_poisson_avg}") - # - fig.tight_layout() - plt.show() - fig.savefig('C:/_data/image.svg', dpi=fig.dpi) - print() - - -def plot_and_integrate(std=0.2, precision=2): - reduce_precision = ReducePrecision(precision=precision) - gaussian_noise = GaussianNoise(std=std, precision=precision) - poisson_noise = PoissonNoise(scale=1 / std, precision=precision) - - def normal_int_fn(p, q): - rp_q = reduce_precision(q) - return np.sign(dirac_delta(reduce_precision(p) - rp_q)) * gaussian_noise.pdf(p, loc=rp_q) - - def poisson_int_fn(p, q): - rp_q = reduce_precision(q) - return np.sign(dirac_delta(reduce_precision(p) - rp_q)) * poisson_noise.pdf(p, rate=abs(rp_q)) - - def p_threshold(q, a=-np.inf, b=np.inf): - return [ - reduce_precision(q) - reduce_precision.precision_width / 2, - reduce_precision(q) + reduce_precision.precision_width / 2 - ] - # return [ - # np.maximum(float(reduce_precision(q) - reduce_precision.step_width / 2), a), - # np.minimum(float(reduce_precision(q) + reduce_precision.step_width / 2), b) - # ] - - k = int(reduce_precision.precision.data) + 1 - num_line_domain = [-1.01, 1.01] - num_line = np.linspace(*num_line_domain, 250, dtype=float) - - plt.figure() - fig, axes = plt.subplots(k + 1, 1) - fig.set_size_inches(5, 7.5) - fig.set_dpi(500) - plt.subplot(k + 1, 1, 1) - - # n_int_normal_correct = integrate.nquad(normal_int_fn, [p_threshold, num_line_domain])[0] / np.sum(np.abs(num_line_domain)) - # n_int_poisson_correct = integrate.nquad(poisson_int_fn, [p_threshold, num_line_domain])[0] / np.sum(np.abs(num_line_domain)) - - normal_correct = [] - poisson_correct = [] - for i in num_line: - normal_correct.append( - integrate.quad(gaussian_noise.pdf, *p_threshold(i, -1., 1.), args=(reduce_precision(i),))[0]) - poisson_correct.append( - integrate.quad(poisson_noise.pdf, *p_threshold(i, -1., 1.), args=(reduce_precision(i),))[0]) - - # plt.plot(num_line, abs(reduce_precision(num_line)), label="abs reduce_precision") - plt.plot(num_line, normal_correct, label="normal_correct") - plt.plot(num_line, poisson_correct, label="poisson_correct") - - plt.ylim(-0.01, 1.1) - plt.legend() - for i in range(k): - plt.subplot(k + 1, 1, i + 2) - plt.plot(num_line, normal_int_fn(num_line, i * reduce_precision.precision_width), label=f"normal_int {i}") - plt.plot(num_line, poisson_int_fn(num_line, i * reduce_precision.precision_width), label=f"poisson_int {i}") - plt.plot(num_line, normal_int_fn(num_line, -i * reduce_precision.precision_width), label=f"normal_int {-i}") - plt.plot(num_line, poisson_int_fn(num_line, -i * reduce_precision.precision_width), label=f"poisson_int {-i}") - plt.ylim(-0.01, 1.1) - plt.legend() - fig.tight_layout() - plt.show() - # fig.savefig('image.svg', dpi=fig.dpi) - - pdf_normal_correct = 0 - cdf_normal_correct = 0 - pdf_poisson_correct = 0 - cdf_poisson_correct = 0 - cdf_domain = np.linspace(-1, 1, 2 * precision + 1, dtype=float) - print(precision) - print(cdf_domain) - print(reduce_precision(cdf_domain)) - for i in cdf_domain: - min_i = i - reduce_precision.precision_width / 2 - max_i = i + reduce_precision.precision_width / 2 - if np.isclose(i, 1.): - max_i = 1. - if np.isclose(i, -1.): - min_i = -1. - - # print(f"N {i:0.4f}: {float(gaussian_noise.cdf(max_i, mean=i) - gaussian_noise.cdf(min_i, mean=i)):0.5f}") - pdf_normal_correct += integrate.quad(gaussian_noise.pdf, min_i, max_i, args=(reduce_precision(i),))[0] - cdf_normal_correct += gaussian_noise.cdf(max_i, mean=reduce_precision(i)) - cdf_normal_correct -= gaussian_noise.cdf(min_i, mean=reduce_precision(i)) - - pdf_poisson_correct += integrate.quad(poisson_noise.pdf, min_i, max_i, args=(reduce_precision(i),))[0] - print(f"P {np.isclose(i, reduce_precision(i))}, {i:0.4f}" - f" : [{max(abs(max_i), abs(min_i)):0.4f}, {min(abs(max_i), abs(min_i)):0.4f}]" - f" : {float(poisson_noise.cdf(max(abs(max_i), abs(min_i)), rate=i)):0.5f}" - f" - {float(poisson_noise.cdf(min(abs(max_i), abs(min_i)), rate=i)):0.5f}" - f" = {float(poisson_noise.cdf(max(abs(max_i), abs(min_i)), rate=i) - poisson_noise.cdf(min(abs(max_i), abs(min_i)), rate=i)):0.5f}") - - if np.isclose(i, 0.): - cdf_poisson_correct += poisson_noise.cdf(max(abs(max_i), abs(min_i)), rate=i) - else: - cdf_poisson_correct += poisson_noise.cdf(max(abs(max_i), abs(min_i)), rate=i) - cdf_poisson_correct -= poisson_noise.cdf(min(abs(max_i), abs(min_i)), rate=i) - - pdf_normal_correct /= cdf_domain.size - 1 - cdf_normal_correct /= cdf_domain.size - 1 - pdf_poisson_correct /= cdf_domain.size - 1 - cdf_poisson_correct /= cdf_domain.size - 1 - - print( - f"normal_correct" - f" - expected: {1 - gaussian_noise.leakage:0.6f}" - f", got numerically: {np.mean(normal_correct):0.6f}" - # f", got integrally: {n_int_normal_correct:0.6f}" - f", got pdf: {pdf_normal_correct:0.6f}" - f", got cdf: {cdf_normal_correct:0.6f}" - ) - print( - f"poisson_correct" - f" - expected: {1 - poisson_noise.leakage:0.6f}" - f", got numerically: {np.mean(poisson_correct):0.6f}" - # f", got integrally: {n_int_poisson_correct:0.6f}" - f", got pdf: {pdf_poisson_correct:0.6f}" - f", got cdf: {cdf_poisson_correct:0.6f}" - ) - print( - f"normal_leakage" - f" - expected: {gaussian_noise.leakage:0.6f}" - f", got numerically: {1 - np.mean(normal_correct):0.6f}" - f", got pdf: {1 - pdf_normal_correct:0.6f}" - f", got cdf: {1 - cdf_normal_correct:0.6f}" - ) - print( - f"poisson_leakage" - f" - expected: {poisson_noise.leakage:0.6f}" - f", got numerically: {1 - np.mean(poisson_correct):0.6f}" - f", got pdf: {1 - pdf_poisson_correct:0.6f}" - f", got cdf: {1 - cdf_poisson_correct:0.6f}" - ) - print(sorted(set(poisson_correct))) - main(std=std, precision=precision) - - -def plot_poisson_leakage(): - max_bit = 6 - precision = np.linspace(2 ** 0, 2 ** max_bit, 2 ** max_bit) - scale = np.linspace(0, 20, 210) - leakage = np.zeros((precision.size, scale.size)) - for i, p in enumerate(precision): - for j, s in enumerate(scale): - leakage[i][j] = PoissonNoise.staticmethod_leakage(scale=s, precision=p) - - scale, precision = np.meshgrid(scale, precision) - savemat("../_data/plot_poisson_leakage.mat", { - "precision": precision, - "scale": scale, - "leakage": leakage, - }) - plt.contourf(scale, precision, leakage, 100) - plt.colorbar() - plt.show() - - -def plot_normal_leakage(): - max_bit = 3 - precision = np.linspace(2 ** 0, 2 ** max_bit, 2 ** max_bit) - std = np.linspace(0, 1, 500) - leakage = np.zeros((precision.size, std.size)) - for i, p in enumerate(precision): - for j, s in enumerate(std): - leakage[i][j] = GaussianNoise.calc_leakage(std=s, precision=p) - - std, precision = np.meshgrid(std, precision) - savemat("../_data/plot_normal_leakage.mat", { - "precision": precision, - "std": std, - "leakage": leakage, - }) - plt.contourf(std, precision, leakage, 100) - plt.colorbar() - plt.show() - - -@torch.no_grad() -def plot_layer_leakage(): - max_bit = 3 - precision = np.linspace(2 ** 0, 2 ** max_bit, 2 ** max_bit) - std = np.linspace(0, 3, 30) - scale = np.linspace(0, 5, 50) - - rp = ReducePrecision(precision=1) - gn = GaussianNoise(std=1, precision=1) - pn = PoissonNoise(scale=1, precision=1) - - layer = Sequential( - rp, - gn, - pn, - # Identity(), - # pn, - # gn, - rp, - ) - - leakage = np.zeros((precision.size, std.size, scale.size)) - for i, p in enumerate(precision): - for j, s in enumerate(std): - print(f"{i}, {j}") - for k, sc in enumerate(scale): - # rp.precision.data = torch.tensor(p) - gn.std.data = torch.tensor(s) - pn.precision.data = torch.tensor(p) - pn.scale.data = torch.tensor(sc) - - temp = torch.rand((2000, 2000)) - leakage[i][j][k] = float(calculate_leakage(rp(temp), layer(temp))) - - savemat("../_data/plot_layer_leakage.mat", { - "precision": precision, - "std": std, - "scale": scale, - "leakage": leakage, - }) - # std, precision, scale = np.meshgrid(std, precision) - # - # plt.contourf(std, precision, leakage, 100) - # plt.colorbar() - # plt.show() - - -if __name__ == '__main__': - main() - # plot_and_integrate() - # plot_poisson_leakage() - # plot_normal_leakage() - # plot_layer_leakage() diff --git a/research/analysis/norm_figures.py b/research/analysis/norm_figures.py deleted file mode 100644 index 6808a54..0000000 --- a/research/analysis/norm_figures.py +++ /dev/null @@ -1,78 +0,0 @@ -import matplotlib -import matplotlib.pyplot as plt -import numpy as np -import torch -import torchvision -import torchvision.transforms as transforms -from torch.utils.data import DataLoader - -from analogvnn.nn.normalize.LPNorm import L1NormM, L2NormM, L1NormWM, L2NormWM - - -def main(): - n_pics = 4 - - trainset = torchvision.datasets.CIFAR10(root='C:/_data/test/dataset', train=True, download=True, - transform=transforms.ToTensor()) - trainloader = DataLoader(trainset, batch_size=n_pics ** 2, shuffle=True, num_workers=2) - - images, labels = iter(trainloader).next() - - norm_l1_norm_images = L1NormM()(images) - norm_l2_norm_images = L2NormM()(images) - norm_l1_norm_w_images = L1NormWM()(images) - norm_l2_norm_w_images = L2NormWM()(images) - - # norm_l1_norm_images *= torch.mean(images) / torch.mean(norm_l1_norm_images) - # norm_l2_norm_images *= torch.mean(images) / torch.mean(norm_l2_norm_images) - # norm_l1_norm_w_images *= torch.mean(images) / torch.mean(norm_l1_norm_w_images) - # norm_l2_norm_w_images *= torch.mean(images) / torch.mean(norm_l2_norm_w_images) - - # norm_l1_norm_images /= torch.max(norm_l1_norm_images) - # norm_l2_norm_images /= torch.max(norm_l2_norm_images) - # norm_l1_norm_w_images /= torch.max(norm_l1_norm_w_images) - # norm_l2_norm_w_images /= torch.max(norm_l2_norm_w_images) - - # norm_l1_norm_images = (norm_l1_norm_images - images) - # norm_l2_norm_images = (norm_l2_norm_images - images) - # norm_l1_norm_w_images = (norm_l1_norm_w_images - images) - # norm_l2_norm_w_images = (norm_l2_norm_w_images - images) - - images = torchvision.utils.make_grid(images, nrow=n_pics, padding=4) - norm_l1_norm_images = torchvision.utils.make_grid(norm_l1_norm_images, nrow=n_pics, padding=4) - norm_l2_norm_images = torchvision.utils.make_grid(norm_l2_norm_images, nrow=n_pics, padding=4) - norm_l1_norm_w_images = torchvision.utils.make_grid(norm_l1_norm_w_images, nrow=n_pics, padding=4) - norm_l2_norm_w_images = torchvision.utils.make_grid(norm_l2_norm_w_images, nrow=n_pics, padding=4) - - npimg = torch.cat(( - images, - norm_l1_norm_images, - norm_l2_norm_images, - norm_l1_norm_w_images, - norm_l2_norm_w_images, - ), dim=2).numpy() - plt.imshow(np.transpose(npimg, (1, 2, 0))) - plt.show() - # imshow(to_image(images)) - - -def create_text(): - matplotlib.rcParams['pdf.fonttype'] = 42 - matplotlib.rcParams['ps.fonttype'] = 42 - - fig = plt.figure() - ax = fig.add_subplot(111) - for axis in ['top', 'bottom', 'left', 'right']: - ax.spines[axis].set_linewidth(0.75) - - fig.set_dpi(200) - - plt.title("Stochastic Reduce Precision Layer") - plt.xlabel("Input") - plt.ylabel("Output") - fig.savefig(f'C:/_data/image.svg', dpi=200, transparent=True) - - -if __name__ == '__main__': - # main() - create_text() diff --git a/research/analysis/std.py b/research/analysis/std.py deleted file mode 100644 index 920ee28..0000000 --- a/research/analysis/std.py +++ /dev/null @@ -1,36 +0,0 @@ -import numpy as np -import torchvision - -from research.dataloaders.load_vision_dataset import load_vision_dataset - - -def main(): - train_loader, test_loader, input_shape, classes = load_vision_dataset( - dataset=torchvision.datasets.CIFAR10, - path="C:/_data/test", - batch_size=1, - grayscale=False - ) - - sums_x = np.zeros(input_shape) - sums_x2 = np.zeros(input_shape) - for (data, target) in train_loader: - data = np.array(data) - sums_x += data - sums_x2 += np.power(data, 2) - for (data, target) in test_loader: - data = np.array(data) - sums_x += data - sums_x2 += np.power(data, 2) - - n = np.shape(train_loader.dataset.data)[0] + np.shape(train_loader.dataset.data)[0] - mean = sums_x / n - mean = np.mean(mean) - std = np.sqrt(np.sum(sums_x2 / (np.prod(input_shape) * n)) - (mean * mean)) - - print(mean) - print(std) - - -if __name__ == '__main__': - main() diff --git a/research/analysis/tensorboard_reader.py b/research/analysis/tensorboard_reader.py deleted file mode 100644 index 45760b4..0000000 --- a/research/analysis/tensorboard_reader.py +++ /dev/null @@ -1,560 +0,0 @@ -import copy -import json -import math -import os -import time -from pathlib import Path -from typing import List, Dict, Union - -import matplotlib -import numpy as np -import seaborn as seaborn -import torch -import torchvision -from matplotlib import pyplot as plt -from matplotlib.pyplot import figure, close -from torch import nn -from torch.utils.data import DataLoader - -import analogvnn.nn.normalize.Normalize -from analogvnn.nn.noise.GaussianNoise import GaussianNoise -from analogvnn.nn.normalize.Clamp import Clamp, Clamp01 -from analogvnn.nn.precision.ReducePrecision import ReducePrecision -from analogvnn.utils.is_cpu_cuda import is_cpu_cuda -from research.dataloaders.load_vision_dataset import load_vision_dataset - - -def collect_parameters_to_json(path, destination=None): - from tensorboard.plugins.hparams.metadata import SESSION_START_INFO_TAG - from tensorboard.plugins.hparams.plugin_data_pb2 import HParamsPluginData - from tensorflow.python.summary.summary_iterator import summary_iterator - tensorboard_dir = Path(path).joinpath("tensorboard") - all_files: List[Path] = [] - for root, dirs, files in os.walk(tensorboard_dir): - for file in files: - all_files.append(Path(root).joinpath(file)) - parameter_data = {} - # c = False - for i, file in enumerate(all_files): - name = file.parent - if "_" not in str(name.name): - name = name.parent - # c = True - name = name.name - if i % 10 == 0: - print(f"[{i}/{len(all_files)}] Processing {name}...") - if name not in parameter_data: - parameter_data[name] = { - "test_accuracy": {}, - "train_accuracy": {}, - "test_loss": {}, - "train_loss": {}, - "parameter": {}, - "raw": [], - } - - this_data = parameter_data[name] - for event in summary_iterator(str(file)): - for value in event.summary.value: - if value.tag == 'Accuracy/test': - this_data["test_accuracy"][int(event.step)] = value.simple_value - continue - if value.tag == 'Loss/test': - this_data["train_accuracy"][int(event.step)] = value.simple_value - continue - if value.tag == 'Accuracy/train': - this_data["test_loss"][int(event.step)] = value.simple_value - continue - if value.tag == 'Loss/train': - this_data["train_loss"][int(event.step)] = value.simple_value - continue - if value.tag == SESSION_START_INFO_TAG: - ssi = HParamsPluginData() - ssi.ParseFromString(value.metadata.plugin_data.content) - hparams = dict(ssi.session_start_info.hparams) - for k in hparams: - hparams[k] = hparams[k].ListFields()[0][1] - this_data["parameter"] = hparams - # this_data["raw"].append(event) - - # if c: - # break - json_filename = f"{tensorboard_dir.parent.name}_data.json" - if destination is None: - file_path = tensorboard_dir.parent.joinpath(json_filename) - else: - file_path = Path(destination).joinpath(json_filename) - - with open(file_path, "w") as file: - file.write(json.dumps(parameter_data)) - - return file_path - - -def to_title_case(string: str): - return " ".join([(x[0].upper() + x[1:].lower()) for x in string.split("_")]) - - -def create_violin_figure(json_file_path, order_by, size_factor=2.85, int_index=False): - matplotlib.rcParams['pdf.fonttype'] = 42 - matplotlib.rcParams['ps.fonttype'] = 42 - - json_file_path = Path(json_file_path) - with open(json_file_path, "r") as file: - run_data: Dict[str, Dict[str, Union[float, str, Dict[str, Union[str, float]]]]] = json.loads(file.read()) - - max_accuracies: Dict[str, float] = {} - parameters_map: Dict[str, Dict[str, Union[str, float]]] = {} - for key, value in run_data.items(): - # key = key[key.find("_") + 1:] - # if key in max_accuracies: - # max_accuracies[key] = max(*value["test_accuracy"].values(), max_accuracies[key]) - # else: - max_accuracies[key] = max(value["test_accuracy"].values()) - parameters_map[key] = value["parameter"] - - if not (isinstance(order_by, list) or isinstance(order_by, tuple)): - order_by = (order_by,) - - plot_data = { - "x": [], - "y": [], - "hue": [] if len(order_by) > 1 else None, - "hue_order": [] if len(order_by) > 1 else None, - } - plot_data_m = copy.deepcopy(plot_data) - plot_data_fm = copy.deepcopy(plot_data) - for key, value in max_accuracies.items(): - parameters = parameters_map[key] - if not all([x in parameters for x in order_by]): - continue - - x_value = parameters[order_by[0]] - if int_index: - x_value = float("inf") if x_value == "None" else float(x_value) - # x_value = "Digital" if x_value == "None" else f"{int(math.log2(int(x_value)))}-bits" - plot_data["x"].append(x_value) - - if len(order_by) > 1: - hue = parameters[order_by[1]] - if int_index: - hue = float("inf") if hue == "None" else float(hue) - plot_data["hue"].append(hue) - - plot_data["y"].append(value * 100) - for k, v in plot_data.items(): - if k == "hue_order": - continue - if v is None: - continue - if parameters["dataset"] == "MNIST": - plot_data_m[k].append(v[-1]) - elif parameters["dataset"] == "FashionMNIST": - plot_data_fm[k].append(v[-1]) - - if len(order_by) > 1: - plot_data["hue_order"] = sorted(list(set(plot_data["hue"]))) - if "Identity" in plot_data["hue_order"]: - plot_data["hue_order"].remove("Identity") - plot_data["hue_order"].insert(0, "Identity") - if "None" in plot_data["hue_order"]: - plot_data["hue_order"].remove("None") - plot_data["hue_order"].insert(0, "None") - - plot_data_m["hue_order"] = plot_data["hue_order"] - plot_data_fm["hue_order"] = plot_data["hue_order"] - - fig = figure() - ax = fig.add_subplot(111) - for axis in ['top', 'bottom', 'left', 'right']: - ax.spines[axis].set_linewidth(0.75) - # 3.25 - # 1.85 - fig_size = [6.5 / 5, 1.75] - if isinstance(size_factor, tuple): - fig_size[0] *= size_factor[0] - fig_size[1] *= size_factor[1] - else: - fig_size[0] *= size_factor - fig_size[1] *= size_factor - - fig.set_size_inches(*fig_size) - fig.set_dpi(200) - hh = 0.6 - color_palette = seaborn.husl_palette(h=hh, l=0.7) - seaborn.violinplot(**plot_data, cut=0, palette=color_palette, inner=None, linewidth=0.1) - color_palette = seaborn.husl_palette(h=hh, l=0.45) - seaborn.stripplot(**plot_data_fm, palette=color_palette, linewidth=0.1, size=3, jitter=1 / 3, dodge=True) - color_palette = seaborn.husl_palette(h=hh, l=0.65) - seaborn.stripplot(**plot_data_m, palette=color_palette, linewidth=0.1, size=3, jitter=1 / 3, dodge=True) - plt.yticks(np.arange(0, 101, 25)) - plt.ylim([0, 100]) - plt.xlabel(to_title_case(order_by[0]).replace(" W", " [W]").replace(" Y", " [Y]")) - # plt.ylabel("Accuracy") - if len(order_by) > 1: - plt.legend(plot_data["hue_order"], title=to_title_case(order_by[1]).replace(" W", " [W]").replace(" Y", " [Y]")) - # plt.title(json_file_path.name) - fig.tight_layout() - plt.show() - # fig.savefig(f'_data/{timestamp}_{json_file_path.name.replace(".json", "")}_{"_".join(order_by)}_image.svg', - # dpi=fig.dpi, transparent=True) - # fig.savefig(f'_data/{timestamp}_{json_file_path.name.replace(".json", "")}_{"_".join(order_by)}_image.png', - # dpi=fig.dpi, transparent=True) - close('all') - - -def create_line_figure(json_file_path, order_by, size_factor=2.85): - matplotlib.rcParams['pdf.fonttype'] = 42 - matplotlib.rcParams['ps.fonttype'] = 42 - - json_file_path = Path(json_file_path) - with open(json_file_path, "r") as file: - run_data: Dict[str, Dict[str, Union[float, str, Dict[str, Union[str, float]]]]] = json.loads(file.read()) - - max_accuracies: Dict[str, List[float]] = {} - parameters_map: Dict[str, Dict[str, Union[str, float]]] = {} - for key, value in run_data.items(): - # key = key[key.find("_") + 1:] - # if key in max_accuracies: - # max_accuracies[key] = max(*value["test_accuracy"].values(), max_accuracies[key]) - # else: - max_accuracies[key] = value["test_accuracy"].values() - parameters_map[key] = value["parameter"] - - if not (isinstance(order_by, list) or isinstance(order_by, tuple)): - order_by = (order_by,) - plot_data = { - "x": [], - "y": [], - "hue": [], - # "hue_order": [] if len(order_by) > 1 else None, - } - for key, value in max_accuracies.items(): - parameters = parameters_map[key] - if not all([x in parameters for x in order_by]): - continue - # print(parameter) - for epoch, accuracy in enumerate(np.array(list(value)) * 100): - plot_data["x"].append(epoch + 1) - plot_data["y"].append(accuracy) - plot_data["hue"].append(", ".join([parameters[x] for x in order_by])) - - # if len(order_by) > 1: - # plot_data["hue_order"] = list(sorted(set(plot_data["hue"]))) - # if "Identity" in plot_data["hue_order"]: - # plot_data["hue_order"].remove("Identity") - # plot_data["hue_order"].insert(0, "Identity") - # if "None" in plot_data["hue_order"]: - # plot_data["hue_order"].remove("None") - # plot_data["hue_order"].insert(0, "None") - - fig = figure() - ax = fig.add_subplot(111) - for axis in ['top', 'bottom', 'left', 'right']: - ax.spines[axis].set_linewidth(0.75) - # 3.25 - # 1.85 - fig.set_size_inches(*[x * size_factor for x in [1.857, 1.75]]) - fig.set_dpi(200) - seaborn.lineplot(**plot_data, palette="Set2", linewidth=1.5) - # plt.yticks(np.arange(0, 101, 20)) - plt.xticks(np.arange(1, 11, 1)) - # plt.ylim([0, 100]) - plt.xlabel("Epochs") - plt.ylabel("Accuracy") - # if len(order_by) > 0: - # plt.legend(plot_data["hue_order"], title=to_title_case(order_by).replace(" W", " [W]").replace(" Y", " [Y]")) - # plt.title(json_file_path.name) - fig.tight_layout() - plt.show() - # fig.savefig(f'_data/{timestamp}_{json_file_path.name.replace(".json", "")}_{order_by}_image.svg', - # dpi=fig.dpi) - # fig.savefig(f'_data/{timestamp}_{json_file_path.name.replace(".json", "")}_{order_by}_image.png', - # dpi=fig.dpi) - close('all') - - -def calculate_snr_signal(signal, noise_signal): - t2 = torch.tensor(2, device=is_cpu_cuda.device) - s = torch.sum(torch.pow(signal, t2)) - n = torch.sum(torch.pow(torch.abs(signal) - torch.abs(noise_signal), t2)) - return s / n - - -dp_calculate_snr = { - "0.2_4_MNIST": [11.088152885437012, 8.456015586853027], - "0.2_16_MNIST": [175.8230438232422, 130.319091796875], - "0.2_64_MNIST": [2672.276123046875, 1898.3907470703125], - "0.5_4_MNIST": [3.2126624584198, 2.893296241760254], - "0.5_16_MNIST": [51.24656295776367, 44.03767013549805], - "0.5_64_MNIST": [807.6673583984375, 661.8540649414062], - "0.8_4_MNIST": [0.4601576328277588, 0.6117348670959473], - "0.8_16_MNIST": [7.358730792999268, 6.856428623199463], - "0.8_64_MNIST": [117.48218536376953, 104.25345611572266], - - "0.2_4_FashionMNIST": [17.030637741088867, 11.329060554504395], - "0.2_16_FashionMNIST": [275.156005859375, 166.92759704589844], - "0.2_64_FashionMNIST": [4364.8115234375, 2605.60400390625], - "0.5_4_FashionMNIST": [5.585911273956299, 4.426943778991699], - "0.5_16_FashionMNIST": [89.73049926757812, 60.25444030761719], - "0.5_64_FashionMNIST": [1431.469482421875, 930.8984985351562], - "0.8_4_FashionMNIST": [0.8391902446746826, 1.1593579053878784], - "0.8_16_FashionMNIST": [13.434399604797363, 10.254481315612793], - "0.8_64_FashionMNIST": [214.82745361328125, 147.53366088867188], - - "0.2_4_Weights": [24.76962661743164, 13.233047485351562], - "0.5_4_Weights": [9.46220588684082, 5.732192516326904], - "0.8_4_Weights": [1.7913140058517456, 1.938230037689209], - "0.2_16_Weights": [370.3149719238281, 188.97683715820312], - "0.5_16_Weights": [138.59751892089844, 72.18633270263672], - "0.8_16_Weights": [22.993026733398438, 13.150540351867676], - "0.2_64_Weights": [5829.53662109375, 2944.764404296875], - "0.5_64_Weights": [2171.158203125, 1096.6651611328125], - "0.8_64_Weights": [349.09698486328125, 179.90565490722656], -} - - -def calculate_snr(leakage, precision, dataset): - global dp_calculate_snr - precision = int(precision) - leakage = float(leakage) - pp = "_".join([str(x) for x in [leakage, precision, dataset]]) - if pp in dp_calculate_snr: - return dp_calculate_snr[pp] - - rp = ReducePrecision(precision=precision) - gn = GaussianNoise(leakage=leakage, precision=precision) - cl = Clamp01() - de = is_cpu_cuda.device - - average_response_analog = [] - average_response_digital = [] - if dataset == "MNIST": - torch_dataset = torchvision.datasets.MNIST - elif dataset == "FashionMNIST": - torch_dataset = torchvision.datasets.FashionMNIST - elif dataset == "Weights": - cl = Clamp() - torch_dataset = None - train_loader = [] - test_loader = [] - weight_distribution = lambda: (nn.init.uniform_(torch.zeros(100, 100), a=-1, b=1)) - for i in range(2000): - train_loader.append((weight_distribution(), None)) - else: - raise NotImplemented - - if torch_dataset is not None: - train_loader, test_loader, input_shape, classes = load_vision_dataset( - dataset=torch_dataset, - path="../_data", - batch_size=128, - is_cuda=is_cpu_cuda.is_cuda - ) - - def run_calculate(loader): - if isinstance(loader, DataLoader): - # noinspection PyTypeChecker - dataset_size = len(loader.dataset) - else: - dataset_size = len(loader) - - for batch_idx, (data, target) in enumerate(loader): - data = data.to(de) - data_ay = gn(rp(cl(data))) - data_y = rp(cl(gn(data_ay))) - average_response_analog.append(calculate_snr_signal(data, data_ay)) - average_response_digital.append(calculate_snr_signal(data, data_y)) - - if dataset == "Weights": - print_mod = int(dataset_size / 5) - else: - print_mod = int(dataset_size / (len(data) * 5)) - if print_mod > 0 and batch_idx % print_mod == 0 and batch_idx > 0: - print( - f'{pp}:[{batch_idx * len(data)}/{dataset_size} ({100. * batch_idx / len(loader):.0f}%)]: {sum(average_response_analog) / len(average_response_analog):.4f}, {sum(average_response_digital) / len(average_response_digital):.4f}') - - run_calculate(train_loader) - run_calculate(test_loader) - - average_response_analog = float(sum(average_response_analog) / len(average_response_analog)) - average_response_digital = float(sum(average_response_digital) / len(average_response_digital)) - dp_calculate_snr[pp] = (average_response_analog, average_response_digital) - print(json.dumps(dp_calculate_snr)) - return dp_calculate_snr[pp] - - -def create_scatter_figure(json_file_path, size_factor=2.85): - matplotlib.rcParams['pdf.fonttype'] = 42 - matplotlib.rcParams['ps.fonttype'] = 42 - - json_file_path = Path(json_file_path) - with open(json_file_path, "r") as file: - run_data: Dict[str, Dict[str, Union[float, str, Dict[str, Union[str, float]]]]] = json.loads(file.read()) - - max_accuracies: Dict[str, float] = {} - parameters_map: Dict[str, Dict[str, Union[str, float]]] = {} - for key, value in run_data.items(): - # key = key[key.find("_") + 1:] - # if key in max_accuracies: - # max_accuracies[key] = max(*value["test_accuracy"].values(), max_accuracies[key]) - # else: - max_accuracies[key] = max(value["test_accuracy"].values()) - parameters_map[key] = value["parameter"] - - plot_data = { - "x": [], - "y": [], - "hue": [], - # "hue_order": [] if len(order_by) > 1 else None, - "style": [], - } - flip = False - which_snr = 1 - m = 0 - fm = 0 - for key, value in max_accuracies.items(): - parameters = parameters_map[key] - if not all([x in parameters for x in ["leakage_y", "precision_y", "leakage_w", "precision_w", ]]): - continue - # if not (parameter["dataset"] == "FashionMNIST"): - # m = max(m, value) - # continue - # fm = max(fm, value) - # if not (parameter["leakage_w"] == "0.2"): - # continue - # if not (parameter["precision_y"] == "16"): - # continue - # if not (parameter["optimiser_class"] == "StochasticReducePrecisionOptimizer"): - # continue - # if not (parameter["model_class"] == "Linear3"): - # continue - # if not (parameter["activation_class"] == "Tanh"): - # continue - # if not (parameter["activation_class"] == "LeakyReLU"): - # continue - - # print(parameter) - x_value = 10 * math.log10( - calculate_snr(parameters["leakage_y"], parameters["precision_y"], parameters["dataset"])[1]) - hue = 10 * math.log10(calculate_snr(parameters["leakage_w"], parameters["precision_w"], "Weights")[0]) - plot_data["y"].append(value * 100) - if flip: - plot_data["x"].append(x_value) - plot_data["hue"].append(hue) - else: - plot_data["x"].append(hue) - plot_data["hue"].append(x_value) - plot_data["style"].append(parameters["dataset"]) - - print(m) - print(fm) - # if len(order_by) > 1: - # plot_data["hue_order"] = sorted(list(set(plot_data["hue"]))) - # if "Identity" in plot_data["hue_order"]: - # plot_data["hue_order"].remove("Identity") - # plot_data["hue_order"].insert(0, "Identity") - # if "None" in plot_data["hue_order"]: - # plot_data["hue_order"].remove("None") - # plot_data["hue_order"].insert(0, "None") - - fig = figure() - ax = fig.add_subplot(111) - for axis in ['top', 'bottom', 'left', 'right']: - ax.spines[axis].set_linewidth(0.75) - # 3.25 - # 1.85 - fig_size = [2, 1.75] - if isinstance(size_factor, tuple): - fig_size[0] *= size_factor[0] - fig_size[1] *= size_factor[1] - else: - fig_size[0] *= size_factor - fig_size[1] *= size_factor - - fig.set_size_inches(*fig_size) - fig.set_dpi(200) - # cmap = seaborn.color_palette("flare", len(set(plot_data["hue"])), as_cmap=True) - cmap = seaborn.dark_palette("#69d", n_colors=len(set(plot_data["hue"])), reverse=True, as_cmap=True) - seaborn.lineplot(**plot_data, palette=cmap, linewidth=0.75, ci=1) - plt.yticks(np.arange(0, 101, 20)) - # plt.xticks(np.arange(0, 41, 7.5)) - plt.ylim([0, 100]) - norm = analogvnn.nn.normalize.Normalize.Normalize(vmin=min(plot_data["hue"]), vmax=max(plot_data["hue"])) - sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm) - cbar = plt.colorbar( - sm, - # ticks=np.round(np.linspace(min(plot_data["hue"]), max(plot_data["hue"]), len(set(plot_data["hue"])))), - ) - if flip: - plt.xlabel("SNR [Y] (dB)") - cbar.ax.set_ylabel("SNR [W] (dB)") - else: - plt.xlabel("SNR [W] (dB)") - cbar.ax.set_ylabel("SNR [Y] (dB)") - - fig.tight_layout() - plt.show() - # fig.savefig(f'{timestamp}_{json_file_path.name.replace(".json", "")}_snr_image.svg', - # dpi=fig.dpi, transparent=True) - # fig.savefig(f'{timestamp}_{json_file_path.name.replace(".json", "")}_snr_image.png', - # dpi=fig.dpi, transparent=True) - close('all') - - -if __name__ == '__main__': - data_folder = Path("C:/X/hyperparameter json") - timestamp = int(time.time()) - - norm_order_by = ("norm_class_w", "norm_class_y") - precision_order_by = ("precision_w", "precision_y") - leakage_order_by = ("leakage_w", "leakage_y") - - # for i in os.listdir(data_folder): - # collect_parameters_to_json(data_folder.joinpath(i), data_folder) - - # aa = 1.5 - # for i in [ - # ("tensorboard_cleo_run_1_data.json", "norm_class_w", (aa*2, aa/1.75)), - # ("tensorboard_cleo_run_3_data.json", "leakage_w", (aa*2, aa/1.75)), - # - # ("tensorboard_cleo_run_2_data.json", ("precision_w", "precision_class"), (aa * 1.75, aa * 1.25)), - # # ("tensorboard_cleo_run_2S_F_data.json", precision_order_by, (aa * 2, aa/2), True), - # - # # ("tensorboard_cleo_run_3_data.json", ("leakage_w", "precision_w"), (4, 2), True), - # - # # ("tensorboard_cleo_run_3_data.json", "activation_class"), - # ]: - # create_violin_figure(data_folder.joinpath(i[0]), *i[1:]) - - create_line_figure(data_folder.joinpath("tensorboard_cleo_run_3_data.json"), ("leakage_w", "precision_w")) - aa = 1.75 - create_scatter_figure(data_folder.joinpath("tensorboard_cleo_run_3_data.json"), (1 * aa, 1 * aa)) - - # calculate_snr(0.2, 4, "Weights") - # calculate_snr(0.5, 4, "Weights") - # calculate_snr(0.8, 4, "Weights") - # calculate_snr(0.2, 16, "Weights") - # calculate_snr(0.5, 16, "Weights") - # calculate_snr(0.8, 16, "Weights") - # calculate_snr(0.2, 64, "Weights") - # calculate_snr(0.5, 64, "Weights") - # calculate_snr(0.8, 64, "Weights") - # calculate_snr(0.2, 4, "MNIST") - # calculate_snr(0.5, 4, "MNIST") - # calculate_snr(0.8, 4, "MNIST") - # calculate_snr(0.2, 16, "MNIST") - # calculate_snr(0.5, 16, "MNIST") - # calculate_snr(0.8, 16, "MNIST") - # calculate_snr(0.2, 64, "MNIST") - # calculate_snr(0.5, 64, "MNIST") - # calculate_snr(0.8, 64, "MNIST") - # calculate_snr(0.2, 4, "FashionMNIST") - # calculate_snr(0.5, 4, "FashionMNIST") - # calculate_snr(0.8, 4, "FashionMNIST") - # calculate_snr(0.2, 16, "FashionMNIST") - # calculate_snr(0.5, 16, "FashionMNIST") - # calculate_snr(0.8, 16, "FashionMNIST") - # calculate_snr(0.2, 64, "FashionMNIST") - # calculate_snr(0.5, 64, "FashionMNIST") - # calculate_snr(0.8, 64, "FashionMNIST") diff --git a/research/crc/__init__.py b/research/crc/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/research/crc/_common.py b/research/crc/_common.py deleted file mode 100644 index 0358a6c..0000000 --- a/research/crc/_common.py +++ /dev/null @@ -1,10 +0,0 @@ -def pick_instanceof_module(arr: list, superclass): - result = [] - if superclass is None: - return result - - for i in arr: - if isinstance(i, superclass): - result.append(i) - - return result diff --git a/research/crc/analog_vnn_1.py b/research/crc/analog_vnn_1.py deleted file mode 100644 index f2955d7..0000000 --- a/research/crc/analog_vnn_1.py +++ /dev/null @@ -1,169 +0,0 @@ -import argparse -import json - -import torchvision - -from analogvnn.nn.activation.ELU import ELU -from analogvnn.nn.activation.Gaussian import GeLU -from analogvnn.nn.activation.ReLU import ReLU, LeakyReLU -from analogvnn.nn.activation.SiLU import SiLU -from analogvnn.nn.activation.Tanh import Tanh -from analogvnn.nn.noise.GaussianNoise import GaussianNoise -from analogvnn.nn.noise.PoissonNoise import PoissonNoise -from analogvnn.nn.normalize.Clamp import * -from analogvnn.nn.normalize.LPNorm import L1Norm, L2Norm, L1NormW, L2NormW, L1NormM, L2NormM, L1NormWM, L2NormWM -from analogvnn.nn.precision.ReducePrecision import ReducePrecision -from analogvnn.nn.precision.StochasticReducePrecision import StochasticReducePrecision -from research.crc.analog_vnn_1_model import run_analog_vnn1_model, RunParametersAnalogVNN1 -from research.utils.path_functions import get_relative_path - -analogvnn1_parameters_list = { - "nn_model_params": { - "num_conv_layer": [0, 3], - "num_linear_layer": [1, 2, 3], - "activation_class": [None, ReLU, LeakyReLU, Tanh, ELU, SiLU, GeLU], - - "norm_class": [None, Clamp, L1Norm, L2Norm, L1NormW, L2NormW, L1NormM, L2NormM, L1NormWM, L2NormWM], - "approach": ["default", "no_norm_grad"], - - "precision_class": [None, ReducePrecision, StochasticReducePrecision], - "precision": [2 ** 2, 2 ** 4, 2 ** 6], - - "noise_class": [None, GaussianNoise, PoissonNoise], - "leakage": [0.25, 0.5, 0.75], - }, - "weight_model_params": { - "norm_class": [None, Clamp, L1Norm, L2Norm, L1NormW, L2NormW, L1NormM, L2NormM, L1NormWM, L2NormWM], - - "precision_class": [None, ReducePrecision, StochasticReducePrecision], - "precision": [2 ** 2, 2 ** 4, 2 ** 6], - - "noise_class": [None, GaussianNoise, PoissonNoise], - "leakage": [0.25, 0.5, 0.75], - }, - "dataset": [ - torchvision.datasets.MNIST, - torchvision.datasets.FashionMNIST, - torchvision.datasets.CIFAR10, - # torchvision.datasets.CIFAR100, - ], -} - - -def __select_class(main_object, name, class_list): - value = getattr(main_object, name) - if value is None and None in class_list: - setattr(main_object, name, None) - return None - if value is None and None not in class_list: - raise Exception(f"{name} must be in {class_list}") - - for cl in class_list: - if cl is None: - continue - if value == cl: - setattr(main_object, name, cl) - return cl - if isinstance(value, str): - if value == cl.__name__: - setattr(main_object, name, cl) - return cl - - raise Exception(f"{name} must be in {class_list}") - - -def __check(main_obj, first, second, check_list): - if getattr(main_obj, first) is None: - if getattr(main_obj, second) is not None: - raise Exception(f'{first}=None then {second} must be None') - else: - if getattr(main_obj, second) not in check_list: - raise Exception(f'{second} must be in {check_list}') - - -def run_analog_vnn1(kwargs): - parameters = RunParametersAnalogVNN1() - - for key, value in kwargs.items(): - if hasattr(parameters, key): - setattr(parameters, key, value) - - __select_class(parameters, 'dataset', analogvnn1_parameters_list["dataset"]) - __select_class(parameters, 'activation_class', analogvnn1_parameters_list["nn_model_params"]["activation_class"]) - __select_class(parameters, 'norm_class', analogvnn1_parameters_list["nn_model_params"]["norm_class"]) - __select_class(parameters, 'precision_class', analogvnn1_parameters_list["nn_model_params"]["precision_class"]) - __select_class(parameters, 'noise_class', analogvnn1_parameters_list["nn_model_params"]["noise_class"]) - __select_class(parameters, 'w_norm_class', analogvnn1_parameters_list["weight_model_params"]["norm_class"]) - __select_class(parameters, 'w_precision_class', - analogvnn1_parameters_list["weight_model_params"]["precision_class"]) - __select_class(parameters, 'w_noise_class', analogvnn1_parameters_list["weight_model_params"]["noise_class"]) - - if parameters.num_conv_layer not in analogvnn1_parameters_list["nn_model_params"]["num_conv_layer"]: - raise Exception(f'num_conv_layer must be in {analogvnn1_parameters_list["nn_model_params"]["num_conv_layer"]}') - if parameters.num_linear_layer not in analogvnn1_parameters_list["nn_model_params"]["num_linear_layer"]: - raise Exception( - f'num_linear_layer must be in {analogvnn1_parameters_list["nn_model_params"]["num_linear_layer"]}') - if parameters.approach not in analogvnn1_parameters_list["nn_model_params"]["approach"]: - raise Exception(f'approach must be in {analogvnn1_parameters_list["nn_model_params"]["approach"]}') - - if parameters.norm_class is None and parameters.approach != "default": - raise Exception('norm_class=None then approach must be "default"') - - __check(parameters, "precision_class", "precision", analogvnn1_parameters_list["nn_model_params"]["precision"]) - __check(parameters, "noise_class", "leakage", analogvnn1_parameters_list["nn_model_params"]["leakage"]) - __check(parameters, "w_precision_class", "w_precision", - analogvnn1_parameters_list["weight_model_params"]["precision"]) - __check(parameters, "w_noise_class", "w_leakage", analogvnn1_parameters_list["weight_model_params"]["leakage"]) - - # print(parameter) - run_analog_vnn1_model(parameters) - - -def parser_run_analogvnn1(main_file=None): - parser = argparse.ArgumentParser() - parser.add_argument("--name", type=str, default=None) - parser.add_argument("--timestamp", type=str, default=None) - parser.add_argument("--data_folder", type=str, required=True) - parser.add_argument("--num_conv_layer", type=int, required=True) - parser.add_argument("--num_linear_layer", type=int, required=True) - parser.add_argument("--activation_class", type=str, default=None) - - parser.add_argument("--norm_class", type=str, default=None) - parser.add_argument("--approach", type=str, default="default") - parser.add_argument("--precision_class", type=str, default=None) - parser.add_argument("--precision", type=int, default=None) - parser.add_argument("--noise_class", type=str, default=None) - parser.add_argument("--leakage", type=float, default=None) - - parser.add_argument("--w_norm_class", type=str, default=None) - parser.add_argument("--w_precision_class", type=str, default=None) - parser.add_argument("--w_precision", type=int, default=None) - parser.add_argument("--w_noise_class", type=str, default=None) - parser.add_argument("--w_leakage", type=float, default=None) - - parser.add_argument("--dataset", type=str, default="MNIST") - # parser.add_argument("--epochs", type=int, default=10) - parser.add_argument("--device", type=str, default=None) - - parser.add_argument("--test_logs", action='store_true') - parser.set_defaults(test_logs=False) - parser.add_argument("--test_run", action='store_true') - parser.set_defaults(test_run=False) - parser.add_argument("--tensorboard", action='store_true') - parser.set_defaults(tensorboard=False) - parser.add_argument("--save_data", action='store_true') - parser.set_defaults(save_data=False) - kwargs = vars(parser.parse_known_args()[0]) - print(json.dumps(kwargs)) - print() - - if main_file is None: - main_file = __file__ - - kwargs["data_folder"] = get_relative_path(main_file, kwargs["data_folder"]) - - run_analog_vnn1(kwargs) - - -if __name__ == '__main__': - parser_run_analogvnn1() diff --git a/research/crc/analog_vnn_1_model.py b/research/crc/analog_vnn_1_model.py deleted file mode 100644 index d5f3bcb..0000000 --- a/research/crc/analog_vnn_1_model.py +++ /dev/null @@ -1,475 +0,0 @@ -import dataclasses -import hashlib -import json -import math -from dataclasses import dataclass -from typing import Type, List, Union, Optional - -import numpy as np -import torch.backends.cudnn -import torchinfo -import torchvision -from torch import optim, nn -from torch.nn import Flatten -from torch.optim import Optimizer -from torchvision.datasets import VisionDataset - -from analogvnn.backward.BackwardIdentity import BackwardIdentity -from analogvnn.backward.BackwardUsingForward import BackwardUsingForward -from analogvnn.nn.Linear import Linear -from analogvnn.nn.activation.Activation import Activation -from analogvnn.nn.module.FullSequential import FullSequential -from analogvnn.nn.module.Layer import Layer -from analogvnn.nn.module.Sequential import Sequential -from analogvnn.nn.noise.GaussianNoise import GaussianNoise -from analogvnn.nn.normalize.Normalize import Normalize -from analogvnn.nn.precision.ReducePrecision import ReducePrecision -from analogvnn.nn.precision.StochasticReducePrecision import StochasticReducePrecision -from analogvnn.parameter.PseudoParameter import PseudoParameter -from analogvnn.utils.is_cpu_cuda import is_cpu_cuda -from analogvnn.utils.render_autograd_graph import save_autograd_graph_from_module -from research.crc._common import pick_instanceof_module -from research.dataloaders.load_vision_dataset import load_vision_dataset -from research.utils.data_dirs import data_dirs -from research.utils.path_functions import path_join - -LINEAR_LAYER_SIZES = { - 1: [], - 2: [64], - 3: [128, 64], - 4: [256, 128, 64] -} -CONV_LAYER_SIZES = { - 0: [], - 3: [(1, 32, (3, 3)), (32, 64, (3, 3)), (64, 64, (3, 3))], -} - - -@dataclass -class RunParametersAnalogVNN1: - name: Optional[str] = None - data_folder: Optional[str] = None - - num_conv_layer: Optional[int] = 0 - num_linear_layer: Optional[int] = 1 - activation_class: Optional[Type[Activation]] = None - approach: Optional[str] = "default" - norm_class: Optional[Type[Normalize]] = None - precision_class: Type[Layer] = None - precision: Optional[int] = None - noise_class: Type[Layer] = None - leakage: Optional[float] = None - - w_norm_class: Optional[Type[Normalize]] = None - w_precision_class: Type[Layer] = None - w_precision: Optional[int] = None - w_noise_class: Type[Layer] = None - w_leakage: Optional[float] = None - - optimiser_class: Type[Optimizer] = optim.Adam - optimiser_parameters: dict = None - dataset: Type[VisionDataset] = torchvision.datasets.MNIST - batch_size: int = 128 - epochs: int = 10 - - device: Optional[torch.device] = None - test_logs: bool = False - test_run: bool = False - tensorboard: bool = False - save_data: bool = True - timestamp: str = None - - def __init__(self): - self.optimiser_parameters = {} - - @property - def nn_model_params(self): - return { - "num_conv_layer": self.num_conv_layer, - "num_linear_layer": self.num_linear_layer, - "activation_class": self.activation_class, - "approach": self.approach, - "norm_class": self.norm_class, - "precision_class": self.precision_class, - "precision": self.precision, - "noise_class": self.noise_class, - "leakage": self.leakage, - } - - @property - def weight_model_params(self): - return { - "norm_class": self.w_norm_class, - "precision_class": self.w_precision_class, - "precision": self.w_precision, - "noise_class": self.w_noise_class, - "leakage": self.w_leakage, - } - - @property - def json(self): - return json.loads(json.dumps(dataclasses.asdict(self), default=str)) - - def __repr__(self): - return f"RunParameters({json.dumps(self.json)})" - - -def cross_entropy_loss_accuracy(output, target): - _, preds = torch.max(output.data, 1) - correct = (preds == target).sum().item() - return correct / len(output) - - -class ConvLinearModel(FullSequential): - def __init__( - self, - input_shape, - conv_features_sizes, linear_features_sizes, - activation_class: Type[Activation], - approach: str = "default", - norm_class: Type[Normalize] = None, - precision_class: Type[Union[ReducePrecision, StochasticReducePrecision]] = None, - precision: Union[int, None] = None, - noise_class: Type[Union[GaussianNoise]] = None, - leakage: Union[float, None] = None, - ): - super(ConvLinearModel, self).__init__() - - self.approach = approach - self.input_shape = input_shape - self.conv_features_sizes = conv_features_sizes - self.linear_features_sizes = linear_features_sizes - self.activation_class = activation_class - self.norm_class = norm_class - self.precision_class = precision_class - self.precision = precision - self.noise_class = noise_class - self.leakage = leakage - self.num_conv_layer = len(conv_features_sizes) - self.num_linear_layer = len(linear_features_sizes) - 1 - - temp_x = torch.zeros(input_shape, requires_grad=False) - - self.all_layers: List[nn.Module] = [] - for i in range(len(conv_features_sizes)): - conv_layer = nn.Conv2d( - in_channels=conv_features_sizes[i][0], - out_channels=conv_features_sizes[i][1], - kernel_size=conv_features_sizes[i][2] - ) - - self.add_doa_layers() - self.all_layers.append(conv_layer) - self.add_aod_layers() - - temp_x = conv_layer(temp_x) - - max_pool = nn.MaxPool2d(2, 2) - self.all_layers.append(max_pool) - temp_x = max_pool(temp_x) - - flatten = Flatten(start_dim=1) - self.all_layers.append(flatten) - temp_x = flatten(temp_x) - - for i in range(len(linear_features_sizes)): - linear_layer = Linear(in_features=temp_x.shape[1], out_features=linear_features_sizes[i]) - temp_x = linear_layer(temp_x) - - if activation_class is not None: - activation_class.initialise_(linear_layer.weight) - - self.add_doa_layers() - self.all_layers.append(linear_layer) - self.add_aod_layers() - - self.conv2d_layers = pick_instanceof_module(self.all_layers, nn.Conv2d) - self.max_pool2d_layers = pick_instanceof_module(self.all_layers, nn.MaxPool2d) - self.linear_layers = pick_instanceof_module(self.all_layers, Linear) - self.activation_layers = pick_instanceof_module(self.all_layers, activation_class) - self.norm_layers = pick_instanceof_module(self.all_layers, norm_class) - self.precision_layers = pick_instanceof_module(self.all_layers, precision_class) - self.noise_layers = pick_instanceof_module(self.all_layers, noise_class) - - if approach == "default": - pass - if approach == "use_autograd_graph": - self.backward.use_autograd_graph = True - if approach == "no_norm_grad": - [i.set_backward_function(BackwardIdentity) for i in self.norm_layers] - if approach == "norm_grad_by_forward": - [i.set_backward_function(BackwardUsingForward) for i in self.norm_layers] - - self.add_sequence(*self.all_layers) - - def add_doa_layers(self): - if self.norm_class is not None: - self.all_layers.append(self.norm_class()) - if self.precision_class is not None: - self.all_layers.append(self.precision_class(precision=self.precision)) - if self.noise_class is not None: - self.all_layers.append(self.noise_class(leakage=self.leakage, precision=self.precision)) - - def add_aod_layers(self): - if self.noise_class is not None: - self.all_layers.append(self.noise_class(leakage=self.leakage, precision=self.precision)) - if self.norm_class is not None: - self.all_layers.append(self.norm_class()) - if self.precision_class is not None: - self.all_layers.append(self.precision_class(precision=self.precision)) - - if self.activation_class is not None: - self.all_layers.append(self.activation_class()) - - def set_optimizer(self, optimizer_cls, super_optimizer_cls=None, param_sanitizer=None, **optimiser_parameters): - if super_optimizer_cls is not None: - self.optimizer = super_optimizer_cls( - optimizer_cls=optimizer_cls, - params=self.parameters() if param_sanitizer is None else param_sanitizer(self.parameters()), - **optimiser_parameters - ) - else: - self.optimizer = optimizer_cls( - params=self.parameters() if param_sanitizer is None else param_sanitizer(self.parameters()), - **optimiser_parameters - ) - return self - - def hyperparameters(self): - return { - 'nn_model_class': self.__class__.__name__, - - 'input_shape': self.input_shape, - 'num_conv_layer': self.num_conv_layer, - 'num_linear_layer': self.num_linear_layer, - 'conv_features_sizes': self.conv_features_sizes, - 'linear_features_sizes': self.linear_features_sizes, - 'approach': self.approach, - 'activation_class': self.activation_class.__name__ if self.activation_class is not None else str(None), - 'norm_class_y': self.norm_class.__name__ if self.norm_class is not None else str(None), - 'precision_class_y': self.precision_class.__name__ if self.precision_class is not None else str(None), - 'precision_y': self.precision, - 'noise_class_y': self.noise_class.__name__ if self.noise_class is not None else str(None), - 'leakage_y': self.leakage, - - 'loss_class': self.loss_function.__class__.__name__, - 'accuracy_fn': self.accuracy_function.__name__, - 'optimiser_superclass': self.optimizer.__class__.__name__, - } - - -class WeightModel(Sequential): - def __init__( - self, - norm_class: Type[Normalize] = None, - precision_class: Type[Union[ReducePrecision, StochasticReducePrecision]] = None, - precision: Union[int, None] = None, - noise_class: Type[Union[GaussianNoise]] = None, - leakage: Union[float, None] = None, - ): - super(WeightModel, self).__init__() - self.norm_class = norm_class - self.precision_class = precision_class - self.precision = precision - self.noise_class = noise_class - self.leakage = leakage - - self.all_layers = [] - - if norm_class is not None: - self.all_layers.append(norm_class()) - if precision_class is not None: - self.all_layers.append(precision_class(precision=precision)) - if noise_class is not None: - self.all_layers.append(noise_class(leakage=leakage, precision=precision)) - - self.eval() - if len(self.all_layers) > 0: - self.add_sequence(*self.all_layers) - - def hyperparameters(self): - return { - 'weight_model_class': self.__class__.__name__, - - 'norm_class_w': self.norm_class.__name__ if self.norm_class is not None else str(None), - 'precision_class_w': self.precision_class.__name__ if self.precision_class is not None else str(None), - 'precision_w': self.precision, - 'noise_class_w': self.noise_class.__name__ if self.noise_class is not None else str(None), - 'leakage_w': self.leakage, - } - - -def run_analog_vnn1_model(parameters: RunParametersAnalogVNN1): - torch.backends.cudnn.benchmark = True - - if parameters.device is not None: - is_cpu_cuda.set_device(str(parameters.device)) - device, is_cuda = is_cpu_cuda.is_using_cuda - parameters.device = device - - if parameters.data_folder is None: - raise Exception("data_folder is None") - - if parameters.name is None: - parameters.name = hashlib.sha256(str(parameters).encode("utf-8")).hexdigest() - - print(f"Parameters: {parameters}") - print(f"Name: {parameters.name}") - print(f"Device: {parameters.device}") - - paths = data_dirs(parameters.data_folder, name=parameters.name, timestamp=parameters.timestamp, - tensorboard=parameters.tensorboard) - log_file = path_join(paths.logs, f"{paths.name}_logs.txt") - - print(f"Timestamp: {paths.timestamp}") - print(f"Storage name: {paths.name}") - print() - - print(f"Loading Data...") - train_loader, test_loader, input_shape, classes = load_vision_dataset( - dataset=parameters.dataset, - path=paths.dataset, - batch_size=parameters.batch_size, - is_cuda=is_cuda - ) - - nn_model_params = parameters.nn_model_params - weight_model_params = parameters.weight_model_params - nn_model_params["input_shape"] = input_shape - nn_model_params["conv_features_sizes"] = CONV_LAYER_SIZES[parameters.num_conv_layer] - nn_model_params["linear_features_sizes"] = LINEAR_LAYER_SIZES[parameters.num_linear_layer] + [len(classes)] - del nn_model_params["num_conv_layer"] - del nn_model_params["num_linear_layer"] - - print(f"Creating Models...") - nn_model = ConvLinearModel(**nn_model_params) - weight_model = WeightModel(**weight_model_params) - if parameters.tensorboard: - nn_model.create_tensorboard(paths.tensorboard) - - nn_model.compile(device=device, layer_data=True) - nn_model.loss_function = nn.CrossEntropyLoss() - nn_model.accuracy_function = cross_entropy_loss_accuracy - nn_model.to(device=device) - weight_model.to(device=device) - PseudoParameter.parametrize_module(nn_model, transformation=weight_model) - nn_model.optimizer = parameters.optimiser_class(params=nn_model.parameters()) - - parameter_log = { - 'dataset': parameters.dataset.__name__, - 'batch_size': parameters.batch_size, - 'is_cuda': is_cuda, - - **nn_model.hyperparameters(), - **weight_model.hyperparameters(), - } - - print(f"Creating Log File...") - with open(log_file, "a+") as file: - file.write(json.dumps(parameters.json, sort_keys=True, indent=2) + "\n\n") - file.write(json.dumps(parameter_log, sort_keys=True, indent=2) + "\n\n") - file.write(str(nn_model.optimizer) + "\n\n") - - file.write(str(nn_model) + "\n\n") - file.write(str(weight_model) + "\n\n") - file.write(torchinfo.summary(nn_model, input_size=tuple(input_shape[1:])).__repr__() + "\n\n") - file.write(torchinfo.summary(weight_model, input_size=(1, 1)).__repr__() + "\n\n") - - if parameters.tensorboard: - nn_model.tensorboard.tensorboard.add_text("parameter", json.dumps(parameters.json, sort_keys=True, indent=2)) - - print(f"Saving Graphs...") - save_autograd_graph_from_module(path_join(paths.logs, f"{paths.name}_nn_model"), nn_model, - next(iter(train_loader))[0]) - save_autograd_graph_from_module(path_join(paths.logs, f"{paths.name}_weight_model"), weight_model, - torch.ones((1, 1))) - - if parameters.tensorboard: - nn_model.tensorboard.add_graph(train_loader) - # nn_model.tensorboard.add_graph(train_loader, model=weight_model) - - loss_accuracy = { - "train_loss": [], - "train_accuracy": [], - "test_loss": [], - "test_accuracy": [], - } - - print(f"Starting Training...") - for epoch in range(parameters.epochs): - if parameters.test_logs: - break - - train_loss, train_accuracy = nn_model.train_on( - train_loader, - epoch=epoch, - test_run=parameters.test_run, - ) - test_loss, test_accuracy = nn_model.test_on( - test_loader, - epoch=epoch, - test_run=parameters.test_run - ) - - loss_accuracy["train_loss"].append(train_loss) - loss_accuracy["train_accuracy"].append(train_accuracy) - loss_accuracy["test_loss"].append(test_loss) - loss_accuracy["test_accuracy"].append(test_accuracy) - - str_epoch = str(epoch + 1).zfill(math.ceil(math.log10(parameters.epochs))) - print_str = f'({str_epoch})' \ - f' Training Loss: {train_loss:.4f},' \ - f' Training Accuracy: {100. * train_accuracy:.0f}%,' \ - f' Testing Loss: {test_loss:.4f},' \ - f' Testing Accuracy: {100. * test_accuracy:.0f}%\n' - print(print_str) - - parameter_log["last_epoch"] = epoch - with open(log_file, "a+") as file: - file.write(print_str) - - if parameters.save_data: - torch.save(nn_model.state_dict(), f"{paths.model_data}/{epoch}_state_dict_nn_model") - torch.save(weight_model.state_dict(), f"{paths.model_data}/{epoch}_state_dict_weight_model") - - if parameters.test_run: - break - - if parameters.save_data: - torch.save(str(nn_model), f"{paths.model_data}/{parameter_log['last_epoch']}_str_nn_model") - torch.save(str(weight_model), f"{paths.model_data}/{parameter_log['last_epoch']}_str_weight_model") - - torch.save(parameters.json, f"{paths.model_data}/{parameter_log['last_epoch']}_parameters_json") - torch.save(parameter_log, f"{paths.model_data}/{parameter_log['last_epoch']}_parameter_log") - torch.save(loss_accuracy, f"{paths.model_data}/{parameter_log['last_epoch']}_loss_accuracy") - - torch.save(nn_model.hyperparameters(), - f"{paths.model_data}/{parameter_log['last_epoch']}_hyperparameters_nn_model") - torch.save(weight_model.hyperparameters(), - f"{paths.model_data}/{parameter_log['last_epoch']}_hyperparameters_weight_model") - - torch.save(nn_model.optimizer.state_dict(), - f"{paths.model_data}/{parameter_log['last_epoch']}_state_dict_optimizer") - - if parameters.tensorboard: - parameter_log["input_shape"] = "_".join([str(x) for x in parameter_log["input_shape"]]) - parameter_log["conv_features_sizes"] = "_".join([str(x) for x in parameter_log["conv_features_sizes"]]) - parameter_log["linear_features_sizes"] = "_".join([str(x) for x in parameter_log["linear_features_sizes"]]) - metric_dict = { - "train_loss": loss_accuracy["train_loss"][-1], - "train_accuracy": loss_accuracy["train_accuracy"][-1], - "test_loss": loss_accuracy["test_loss"][-1], - "test_accuracy": loss_accuracy["test_accuracy"][-1], - "min_train_loss": np.min(loss_accuracy["train_loss"]), - "max_train_accuracy": np.max(loss_accuracy["train_accuracy"]), - "min_test_loss": np.min(loss_accuracy["test_loss"]), - "max_test_accuracy": np.max(loss_accuracy["test_accuracy"]), - } - nn_model.tensorboard.tensorboard.add_hparams( - hparam_dict=parameter_log, - metric_dict=metric_dict - ) - - with open(log_file, "a+") as file: - file.write("Run Completed Successfully...") - print() diff --git a/research/crc/parneet.py b/research/crc/parneet.py deleted file mode 100644 index 0d9801d..0000000 --- a/research/crc/parneet.py +++ /dev/null @@ -1,139 +0,0 @@ -import argparse -import json - -from analogvnn.nn.activation.ELU import ELU -from analogvnn.nn.activation.Gaussian import GeLU -from analogvnn.nn.activation.ReLU import ReLU, LeakyReLU -from analogvnn.nn.activation.SiLU import SiLU -from analogvnn.nn.activation.Tanh import Tanh -from analogvnn.nn.noise.GaussianNoise import GaussianNoise -from analogvnn.nn.noise.PoissonNoise import PoissonNoise -from analogvnn.nn.normalize.Clamp import * -from analogvnn.nn.normalize.LPNorm import L1Norm, L2Norm, L1NormW, L2NormW, L1NormM, L2NormM, L1NormWM, L2NormWM -from analogvnn.nn.precision.ReducePrecision import ReducePrecision -from analogvnn.nn.precision.StochasticReducePrecision import StochasticReducePrecision -from research.crc.parneet_model import run_parneet_model, RunParametersParneet -from research.utils.path_functions import get_relative_path - -parneet_parameters_list = { - "nn_model_params": { - "activation_class": [None, ReLU, LeakyReLU, Tanh, ELU, SiLU, GeLU], - "norm_class": [None, Clamp, L1Norm, L2Norm, L1NormW, L2NormW, L1NormM, L2NormM, L1NormWM, L2NormWM], - "precision_class": [None, ReducePrecision, StochasticReducePrecision], - "noise_class": [None, GaussianNoise, PoissonNoise], - }, - "weight_model_params": { - "norm_class": [None, Clamp, L1Norm, L2Norm, L1NormW, L2NormW, L1NormM, L2NormM, L1NormWM, L2NormWM], - "precision_class": [None, ReducePrecision, StochasticReducePrecision], - "noise_class": [None, GaussianNoise, PoissonNoise], - }, -} - - -def __select_class(main_object, name, class_list): - value = getattr(main_object, name) - if value is None and None in class_list: - setattr(main_object, name, None) - return None - if value is None and None not in class_list: - raise Exception(f"{name} must be in {class_list}") - - for cl in class_list: - if cl is None: - continue - if value == cl: - setattr(main_object, name, cl) - return cl - if isinstance(value, str): - if value == cl.__name__: - setattr(main_object, name, cl) - return cl - - raise Exception(f"{name} must be in {class_list}") - - -def __check(main_obj, first, second): - if getattr(main_obj, first) is None: - if getattr(main_obj, second) is not None: - raise Exception(f'{first}=None then {second} must be None') - else: - if getattr(main_obj, second) is None: - raise Exception(f'{second} must not be None') - - -def run_parneet(kwargs): - parameters = RunParametersParneet() - - for key, value in kwargs.items(): - if hasattr(parameters, key): - setattr(parameters, key, value) - - __select_class(parameters, 'activation_class', parneet_parameters_list["nn_model_params"]["activation_class"]) - __select_class(parameters, 'norm_class', parneet_parameters_list["nn_model_params"]["norm_class"]) - __select_class(parameters, 'precision_class', parneet_parameters_list["nn_model_params"]["precision_class"]) - __select_class(parameters, 'noise_class', parneet_parameters_list["nn_model_params"]["noise_class"]) - __select_class(parameters, 'w_norm_class', parneet_parameters_list["weight_model_params"]["norm_class"]) - __select_class(parameters, 'w_precision_class', parneet_parameters_list["weight_model_params"]["precision_class"]) - __select_class(parameters, 'w_noise_class', parneet_parameters_list["weight_model_params"]["noise_class"]) - - __check(parameters, "precision_class", "precision") - __check(parameters, "noise_class", "leakage") - __check(parameters, "w_precision_class", "w_precision") - __check(parameters, "w_noise_class", "w_leakage") - - # print(parameter) - run_parneet_model(parameters) - - -def parser_run_parneet(main_file=None): - parser = argparse.ArgumentParser() - parser.add_argument("--name", type=str, default=None) - parser.add_argument("--timestamp", type=str, default=None) - parser.add_argument("--data_folder", type=str, required=True) - - parser.add_argument("--activation_class", type=str, default=None) - parser.add_argument("--norm_class", type=str, default=None) - parser.add_argument("--precision_class", type=str, default=None) - parser.add_argument("--precision", type=int, default=None) - parser.add_argument("--noise_class", type=str, default=None) - parser.add_argument("--leakage", type=float, default=None) - - parser.add_argument("--w_norm_class", type=str, default=None) - parser.add_argument("--w_precision_class", type=str, default=None) - parser.add_argument("--w_precision", type=int, default=None) - parser.add_argument("--w_noise_class", type=str, default=None) - parser.add_argument("--w_leakage", type=float, default=None) - - parser.add_argument("--epochs", type=int, default=RunParametersParneet.epochs) - parser.add_argument("--device", type=str, default=None) - parser.add_argument("--color", type=str, default=str(RunParametersParneet.color)) - parser.add_argument("--batch_size", type=int, default=RunParametersParneet.batch_size) - - parser.add_argument("--test_logs", action='store_true') - parser.set_defaults(test_logs=False) - parser.add_argument("--test_run", action='store_true') - parser.set_defaults(test_run=False) - parser.add_argument("--tensorboard", action='store_true') - parser.set_defaults(tensorboard=False) - parser.add_argument("--save_data", action='store_true') - parser.set_defaults(save_data=False) - kwargs = vars(parser.parse_known_args()[0]) - print(json.dumps(kwargs)) - print() - - if main_file is None: - main_file = __file__ - - kwargs["data_folder"] = get_relative_path(main_file, kwargs["data_folder"]) - if kwargs["color"].lower() == "true": - kwargs["color"] = True - elif kwargs["color"].lower() == "false": - kwargs["color"] = False - else: - raise ValueError("Invalid value for color") - - run_parneet(kwargs) - - -if __name__ == '__main__': - parser_run_parneet() diff --git a/research/crc/parneet_model.py b/research/crc/parneet_model.py deleted file mode 100644 index 32e098c..0000000 --- a/research/crc/parneet_model.py +++ /dev/null @@ -1,424 +0,0 @@ -import dataclasses -import hashlib -import json -import math -from dataclasses import dataclass -from typing import Type, List, Union, Optional - -import numpy as np -import torch.backends.cudnn -import torchinfo -import torchvision -from torch import optim, nn -from torch.nn import Flatten -from torch.optim import Optimizer -from torchvision.datasets import VisionDataset - -from analogvnn.nn.Linear import Linear -from analogvnn.nn.activation.Activation import Activation -from analogvnn.nn.module.FullSequential import FullSequential -from analogvnn.nn.module.Layer import Layer -from analogvnn.nn.noise.GaussianNoise import GaussianNoise -from analogvnn.nn.normalize.Normalize import Normalize -from analogvnn.nn.precision.ReducePrecision import ReducePrecision -from analogvnn.nn.precision.StochasticReducePrecision import StochasticReducePrecision -from analogvnn.parameter.PseudoParameter import PseudoParameter -from analogvnn.utils.is_cpu_cuda import is_cpu_cuda -from analogvnn.utils.render_autograd_graph import save_autograd_graph_from_module -from research.crc._common import pick_instanceof_module -from research.crc.analog_vnn_1_model import WeightModel -from research.dataloaders.load_vision_dataset import load_vision_dataset -from research.utils.data_dirs import data_dirs -from research.utils.path_functions import path_join - - -@dataclass -class RunParametersParneet: - name: Optional[str] = None - data_folder: Optional[str] = None - - activation_class: Optional[Type[Activation]] = None - norm_class: Optional[Type[Normalize]] = None - precision_class: Type[Layer] = None - precision: Optional[int] = None - noise_class: Type[Layer] = None - leakage: Optional[float] = None - - w_norm_class: Optional[Type[Normalize]] = None - w_precision_class: Type[Layer] = None - w_precision: Optional[int] = None - w_noise_class: Type[Layer] = None - w_leakage: Optional[float] = None - - optimiser_class: Type[Optimizer] = optim.Adam - optimiser_parameters: dict = None - dataset: Type[VisionDataset] = torchvision.datasets.CIFAR10 - color: bool = True - batch_size: int = 128 - epochs: int = 200 - - device: Optional[torch.device] = None - test_logs: bool = False - test_run: bool = False - tensorboard: bool = False - save_data: bool = True - timestamp: str = None - - def __init__(self): - self.optimiser_parameters = {} - - @property - def nn_model_params(self): - return { - "activation_class": self.activation_class, - "norm_class": self.norm_class, - "precision_class": self.precision_class, - "precision": self.precision, - "noise_class": self.noise_class, - "leakage": self.leakage, - } - - @property - def weight_model_params(self): - return { - "norm_class": self.w_norm_class, - "precision_class": self.w_precision_class, - "precision": self.w_precision, - "noise_class": self.w_noise_class, - "leakage": self.w_leakage, - } - - @property - def json(self): - return json.loads(json.dumps(dataclasses.asdict(self), default=str)) - - def __repr__(self): - return f"RunParameters({json.dumps(self.json)})" - - -def cross_entropy_loss_accuracy(output, target): - _, preds = torch.max(output.data, 1) - correct = (preds == target).sum().item() - return correct / len(output) - - -class ParneetModel(FullSequential): - def __init__( - self, - input_shape, num_classes, - activation_class: Type[Activation], - norm_class: Type[Normalize] = None, - precision_class: Type[Union[ReducePrecision, StochasticReducePrecision]] = None, - precision: Union[int, None] = None, - noise_class: Type[Union[GaussianNoise]] = None, - leakage: Union[float, None] = None, - ): - super(ParneetModel, self).__init__() - - self.input_shape = input_shape - self.num_classes = num_classes - self.activation_class = activation_class - self.norm_class = norm_class - self.precision_class = precision_class - self.precision = precision - self.noise_class = noise_class - self.leakage = leakage - - self.all_layers: List[nn.Module] = [] - - self.add_doa_layers() - self.all_layers.append(nn.Conv2d( - in_channels=self.input_shape[1], - out_channels=48, - kernel_size=(3, 3), - padding=(1, 1) - )) - self.add_aod_layers() - self.add_doa_layers() - self.all_layers.append(nn.Conv2d(in_channels=48, out_channels=48, kernel_size=(3, 3))) - self.add_aod_layers() - self.all_layers.append(nn.MaxPool2d(2, 2)) - self.all_layers.append(nn.Dropout(0.25)) - self.add_doa_layers() - self.all_layers.append(nn.Conv2d(in_channels=48, out_channels=96, kernel_size=(3, 3), padding=(1, 1))) - self.add_aod_layers() - self.add_doa_layers() - self.all_layers.append(nn.Conv2d(in_channels=96, out_channels=96, kernel_size=(3, 3))) - self.add_aod_layers() - self.all_layers.append(nn.MaxPool2d(2, 2)) - self.all_layers.append(nn.Dropout(0.25)) - self.add_doa_layers() - self.all_layers.append(nn.Conv2d(in_channels=96, out_channels=192, kernel_size=(3, 3), padding=(1, 1))) - self.add_aod_layers() - self.add_doa_layers() - self.all_layers.append(nn.Conv2d(in_channels=192, out_channels=192, kernel_size=(3, 3))) - self.add_aod_layers() - self.all_layers.append(nn.MaxPool2d(2, 2)) - self.all_layers.append(nn.Dropout(0.25)) - - self.all_layers.append(Flatten(start_dim=1)) - - self.add_doa_layers() - self.all_layers.append(Linear(in_features=self.get_in_shape(input_shape)[1], out_features=512)) - self.add_aod_layers() - self.all_layers.append(nn.Dropout(0.5)) - self.add_doa_layers() - self.all_layers.append(Linear(in_features=512, out_features=256)) - self.add_aod_layers() - self.all_layers.append(nn.Dropout(0.5)) - self.add_doa_layers() - self.all_layers.append(Linear(in_features=256, out_features=num_classes)) - self.add_aod_layers() - - self.conv2d_layers = pick_instanceof_module(self.all_layers, nn.Conv2d) - self.max_pool2d_layers = pick_instanceof_module(self.all_layers, nn.MaxPool2d) - self.linear_layers = pick_instanceof_module(self.all_layers, Linear) - self.activation_layers = pick_instanceof_module(self.all_layers, self.activation_class) - self.norm_layers = pick_instanceof_module(self.all_layers, norm_class) - self.precision_layers = pick_instanceof_module(self.all_layers, precision_class) - self.noise_layers = pick_instanceof_module(self.all_layers, noise_class) - - for i in self.linear_layers: - self.activation_class.initialise_(i.weight) - - self.add_sequence(*self.all_layers) - - def add_doa_layers(self): - if self.norm_class is not None: - self.all_layers.append(self.norm_class()) - if self.precision_class is not None: - self.all_layers.append(self.precision_class(precision=self.precision)) - if self.noise_class is not None: - self.all_layers.append(self.noise_class(leakage=self.leakage, precision=self.precision)) - - def add_aod_layers(self): - if self.noise_class is not None: - self.all_layers.append(self.noise_class(leakage=self.leakage, precision=self.precision)) - if self.norm_class is not None: - self.all_layers.append(self.norm_class()) - if self.precision_class is not None: - self.all_layers.append(self.precision_class(precision=self.precision)) - - if self.activation_class is not None: - self.all_layers.append(self.activation_class()) - - def get_in_shape(self, input_shape): - with torch.no_grad(): - temp_x = torch.zeros(input_shape, requires_grad=False) - for i in self.all_layers: - # print(temp_x.shape) - temp_t = i.training - i.train(False) - temp_x = i(temp_x) - i.train(temp_t) - return temp_x.shape - - def set_optimizer(self, optimizer_cls, super_optimizer_cls=None, param_sanitizer=None, **optimiser_parameters): - if super_optimizer_cls is not None: - self.optimizer = super_optimizer_cls( - optimizer_cls=optimizer_cls, - params=self.parameters() if param_sanitizer is None else param_sanitizer(self.parameters()), - **optimiser_parameters - ) - else: - self.optimizer = optimizer_cls( - params=self.parameters() if param_sanitizer is None else param_sanitizer(self.parameters()), - **optimiser_parameters - ) - return self - - def hyperparameters(self): - return { - 'nn_model_class': self.__class__.__name__, - - 'input_shape': self.input_shape, - 'num_classes': self.num_classes, - 'activation_class': self.activation_class.__name__ if self.activation_class is not None else str(None), - 'norm_class_y': self.norm_class.__name__ if self.norm_class is not None else str(None), - 'precision_class_y': self.precision_class.__name__ if self.precision_class is not None else str(None), - 'precision_y': self.precision, - 'noise_class_y': self.noise_class.__name__ if self.noise_class is not None else str(None), - 'leakage_y': self.leakage, - - 'loss_class': self.loss_function.__class__.__name__, - 'accuracy_fn': self.accuracy_function.__name__, - 'optimiser_superclass': self.optimizer.__class__.__name__, - } - - -def run_parneet_model(parameters: RunParametersParneet): - print(parameters.color) - torch.backends.cudnn.benchmark = True - - if parameters.device is not None: - is_cpu_cuda.set_device(parameters.device) - device, is_cuda = is_cpu_cuda.is_using_cuda - parameters.device = device - - if parameters.data_folder is None: - raise Exception("data_folder is None") - - if parameters.name is None: - parameters.name = hashlib.sha256(str(parameters).encode("utf-8")).hexdigest() - - print(f"Parameters: {parameters}") - print(f"Name: {parameters.name}") - print(f"Device: {parameters.device}") - - paths = data_dirs(parameters.data_folder, name=parameters.name, timestamp=parameters.timestamp, - tensorboard=parameters.tensorboard) - log_file = path_join(paths.logs, f"{paths.name}_logs.txt") - - print(f"Timestamp: {paths.timestamp}") - print(f"Storage name: {paths.name}") - print() - - print(f"Loading Data...") - train_loader, test_loader, input_shape, classes = load_vision_dataset( - dataset=parameters.dataset, - path=paths.dataset, - batch_size=parameters.batch_size, - is_cuda=is_cuda, - grayscale=not parameters.color - ) - - nn_model_params = parameters.nn_model_params - weight_model_params = parameters.weight_model_params - nn_model_params["input_shape"] = input_shape - nn_model_params["num_classes"] = len(classes) - - print(f"Creating Models...") - nn_model = ParneetModel(**nn_model_params) - weight_model = WeightModel(**weight_model_params) - if parameters.tensorboard: - nn_model.create_tensorboard(paths.tensorboard) - - nn_model.compile(device=device, layer_data=True) - nn_model.loss_function = nn.CrossEntropyLoss() - nn_model.accuracy_function = cross_entropy_loss_accuracy - nn_model.to(device=device) - weight_model.to(device=device) - PseudoParameter.parametrize_module(nn_model, transformation=weight_model) - nn_model.optimizer = parameters.optimiser_class(params=nn_model.parameters()) - - parameter_log = { - 'dataset': parameters.dataset.__name__, - 'batch_size': parameters.batch_size, - 'is_cuda': is_cuda, - 'color': parameters.color, - 'epochs': parameters.epochs, - - **nn_model.hyperparameters(), - **weight_model.hyperparameters(), - } - - print(f"Creating Log File...") - with open(log_file, "a+") as file: - file.write(json.dumps(parameters.json, sort_keys=True, indent=2) + "\n\n") - file.write(json.dumps(parameter_log, sort_keys=True, indent=2) + "\n\n") - file.write(str(nn_model.optimizer) + "\n\n") - - file.write(str(nn_model) + "\n\n") - file.write(str(weight_model) + "\n\n") - file.write(torchinfo.summary(nn_model, input_size=tuple(input_shape[1:])).__repr__() + "\n\n") - file.write(torchinfo.summary(weight_model, input_size=(1, 1)).__repr__() + "\n\n") - - if parameters.tensorboard: - nn_model.tensorboard.tensorboard.add_text("parameter", json.dumps(parameters.json, sort_keys=True, indent=2)) - - print(f"Saving Graphs...") - save_autograd_graph_from_module(path_join(paths.logs, f"{paths.name}_nn_model"), nn_model, - next(iter(train_loader))[0]) - save_autograd_graph_from_module(path_join(paths.logs, f"{paths.name}_weight_model"), weight_model, - torch.ones((1, 1))) - - if parameters.tensorboard: - nn_model.tensorboard.add_graph(train_loader) - # nn_model.tensorboard.add_graph(train_loader, model=weight_model) - - loss_accuracy = { - "train_loss": [], - "train_accuracy": [], - "test_loss": [], - "test_accuracy": [], - } - - print(f"Starting Training...") - for epoch in range(parameters.epochs): - if parameters.test_logs: - break - - train_loss, train_accuracy = nn_model.train_on( - train_loader, - epoch=epoch, - test_run=parameters.test_run, - ) - test_loss, test_accuracy = nn_model.test_on( - test_loader, - epoch=epoch, - test_run=parameters.test_run - ) - - loss_accuracy["train_loss"].append(train_loss) - loss_accuracy["train_accuracy"].append(train_accuracy) - loss_accuracy["test_loss"].append(test_loss) - loss_accuracy["test_accuracy"].append(test_accuracy) - - str_epoch = str(epoch + 1).zfill(math.ceil(math.log10(parameters.epochs))) - print_str = f'({str_epoch})' \ - f' Training Loss: {train_loss:.4f},' \ - f' Training Accuracy: {100. * train_accuracy:.0f}%,' \ - f' Testing Loss: {test_loss:.4f},' \ - f' Testing Accuracy: {100. * test_accuracy:.0f}%\n' - print(print_str) - - parameter_log["last_epoch"] = epoch - with open(log_file, "a+") as file: - file.write(print_str) - - if parameters.save_data: - torch.save(nn_model.state_dict(), f"{paths.model_data}/{epoch}_state_dict_nn_model") - torch.save(weight_model.state_dict(), f"{paths.model_data}/{epoch}_state_dict_weight_model") - - if train_accuracy < 0.125 and epoch >= 9: - break - - if parameters.test_run: - break - - if parameters.save_data: - torch.save(str(nn_model), f"{paths.model_data}/{parameter_log['last_epoch']}_str_nn_model") - torch.save(str(weight_model), f"{paths.model_data}/{parameter_log['last_epoch']}_str_weight_model") - - torch.save(parameters.json, f"{paths.model_data}/{parameter_log['last_epoch']}_parameters_json") - torch.save(parameter_log, f"{paths.model_data}/{parameter_log['last_epoch']}_parameter_log") - torch.save(loss_accuracy, f"{paths.model_data}/{parameter_log['last_epoch']}_loss_accuracy") - - torch.save(nn_model.hyperparameters(), - f"{paths.model_data}/{parameter_log['last_epoch']}_hyperparameters_nn_model") - torch.save(weight_model.hyperparameters(), - f"{paths.model_data}/{parameter_log['last_epoch']}_hyperparameters_weight_model") - - torch.save(nn_model.optimizer.state_dict(), - f"{paths.model_data}/{parameter_log['last_epoch']}_state_dict_optimizer") - - if parameters.tensorboard: - parameter_log["input_shape"] = "_".join([str(x) for x in parameter_log["input_shape"]]) - metric_dict = { - "train_loss": loss_accuracy["train_loss"][-1], - "train_accuracy": loss_accuracy["train_accuracy"][-1], - "test_loss": loss_accuracy["test_loss"][-1], - "test_accuracy": loss_accuracy["test_accuracy"][-1], - "min_train_loss": np.min(loss_accuracy["train_loss"]), - "max_train_accuracy": np.max(loss_accuracy["train_accuracy"]), - "min_test_loss": np.min(loss_accuracy["test_loss"]), - "max_test_accuracy": np.max(loss_accuracy["test_accuracy"]), - } - nn_model.tensorboard.tensorboard.add_hparams( - hparam_dict=parameter_log, - metric_dict=metric_dict - ) - - with open(log_file, "a+") as file: - file.write("Run Completed Successfully...") - print() diff --git a/research/dataloaders/__init__.py b/research/dataloaders/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/research/dataloaders/load_vision_dataset.py b/research/dataloaders/load_vision_dataset.py deleted file mode 100644 index 0ca6a0b..0000000 --- a/research/dataloaders/load_vision_dataset.py +++ /dev/null @@ -1,49 +0,0 @@ -import inspect -from typing import Type - -from torch.utils.data import DataLoader -from torchvision.datasets import VisionDataset -from torchvision.transforms import transforms - - -def get_vision_dataset_transformation(grayscale=True): - if grayscale: - return transforms.Compose([ - transforms.Grayscale(), - transforms.ToTensor(), - # transforms.Normalize(mean=[0.5], std=[0.5]) - ]) - - return transforms.Compose([ - transforms.ToTensor(), - # transforms.Normalize(mean=[0.5], std=[0.5]) - ]) - - -def load_vision_dataset(dataset: Type[VisionDataset], path, batch_size, is_cuda=False, grayscale=True) -> ( - DataLoader, DataLoader, tuple): - dataset_kwargs = { - 'batch_size': batch_size, - 'shuffle': True - } - - if is_cuda: - cuda_kwargs = { - 'num_workers': 1, - 'pin_memory': True, - } - dataset_kwargs.update(cuda_kwargs) - - if "train" not in inspect.getfullargspec(dataset.__init__).args: - raise Exception(f"{dataset} does have a pre split of training data.") - - train_set = dataset(root=path, train=True, download=True, transform=get_vision_dataset_transformation(grayscale)) - train_loader = DataLoader(train_set, **dataset_kwargs) - - test_set = dataset(root=path, train=False, download=True, transform=get_vision_dataset_transformation(grayscale)) - test_loader = DataLoader(test_set, **dataset_kwargs) - - zeroth_element = next(iter(test_loader))[0] - input_shape = list(zeroth_element.shape) - - return train_loader, test_loader, input_shape, tuple(train_set.classes) diff --git a/research/dataloaders/loss_functions.py b/research/dataloaders/loss_functions.py deleted file mode 100644 index 87065f5..0000000 --- a/research/dataloaders/loss_functions.py +++ /dev/null @@ -1,19 +0,0 @@ -import torch -import torch.nn as nn - -nllloss_fn = nn.NLLLoss(reduction='sum') -criterion = nn.CrossEntropyLoss() - - -def nll_loss_fn(output, target): - loss = nllloss_fn(output, target) - pred = output.argmax(dim=1, keepdim=True) - correct = pred.eq(target.view_as(pred)).sum().item() - return loss, loss.item(), correct - - -def cross_entropy_loss_fn(output, target): - loss = criterion(output, target) - _, preds = torch.max(output.data, 1) - correct = (preds == target).sum().item() - return loss, loss.item() * target.shape[0], correct diff --git a/research/main_analogvnn1.py b/research/main_analogvnn1.py deleted file mode 100644 index 18f4f83..0000000 --- a/research/main_analogvnn1.py +++ /dev/null @@ -1,4 +0,0 @@ -from research.crc.analog_vnn_1 import parser_run_analogvnn1 - -if __name__ == '__main__': - parser_run_analogvnn1(__file__) diff --git a/research/main_parneet.py b/research/main_parneet.py deleted file mode 100644 index c12a597..0000000 --- a/research/main_parneet.py +++ /dev/null @@ -1,4 +0,0 @@ -from research.crc.parneet import parser_run_parneet - -if __name__ == '__main__': - parser_run_parneet(__file__) diff --git a/research/parallel_analogvnn1.py b/research/parallel_analogvnn1.py deleted file mode 100644 index 66f9505..0000000 --- a/research/parallel_analogvnn1.py +++ /dev/null @@ -1,420 +0,0 @@ -import argparse -import copy -import hashlib -import inspect -import itertools -import math -import os -import shutil -import subprocess -import time -from collections import OrderedDict -from multiprocessing.pool import ThreadPool -from pathlib import Path - -import torch -import torchvision -from natsort import natsorted, ns - -from analogvnn.nn.normalize.Clamp import * -from analogvnn.nn.normalize.LPNorm import L2Norm -from research.crc.analog_vnn_1 import analogvnn1_parameters_list - - -def prepare_data_folder(folder_path): - folder_path = Path(folder_path) - runtime_path = folder_path.joinpath("runtime") - datasets_path = folder_path.joinpath("datasets") - models_path = folder_path.joinpath("models") - tensorboard_path = folder_path.joinpath("tensorboard") - logs_path = folder_path.joinpath("logs") - - if not folder_path.exists(): - os.mkdir(folder_path) - if not runtime_path.exists(): - os.mkdir(runtime_path) - if not datasets_path.exists(): - os.mkdir(datasets_path) - if not models_path.exists(): - os.mkdir(models_path) - if not tensorboard_path.exists(): - os.mkdir(tensorboard_path) - if not logs_path.exists(): - os.mkdir(logs_path) - - torchvision.datasets.MNIST(root=str(datasets_path.absolute()), download=True) - torchvision.datasets.FashionMNIST(root=str(datasets_path.absolute()), download=True) - torchvision.datasets.CIFAR10(root=str(datasets_path.absolute()), download=True) - torchvision.datasets.CIFAR100(root=str(datasets_path.absolute()), download=True) - - -def run_command(command): - data_folder, command = command - runtime = Path(data_folder).joinpath("runtime") - hash_id = hashlib.sha256(str(command).encode("utf-8")).hexdigest() - timestamp = f"{int(time.time() * 1000)}" - - if "--timestamp" not in command: - command += f" --timestamp {timestamp}" - else: - timestamp = command.split("--timestamp")[-1] - timestamp = timestamp.strip().split(" ")[0] - - if "--name" not in command: - command += f" --name {hash_id}" - else: - hash_id = command.split("--name")[-1] - hash_id = hash_id.strip().split(" ")[0] - - filename = f"{timestamp}_{hash_id}" - out_file = runtime.joinpath(f"{filename}.log") - - with open(out_file, "w+") as out: - out.write(command + "\n") - out.write(f"Running {filename} :: {command}\n\n") - print(f"Running {filename} :: {command}") - - p = subprocess.Popen(command, shell=True, stdout=out, stderr=out) - p.wait() - rc = p.returncode - - out.write(f"\n\n") - if rc == 0: - out.write(f"Success {p.pid} :: {filename} :: {command}") - print(f"Success {p.pid} :: {filename} :: {command}") - else: - out.write(f"Failed {p.pid} :: {filename} :: {rc} :: {command}") - print(f"Failed {p.pid} :: {filename} :: {rc} :: {command}") - - out.write(f"\n\n{rc}") - - -def parameter_checkers(parameters): - if ("norm_class" not in parameters or parameters["norm_class"] is None) and "approach" in parameters: - if parameters["approach"] != "default": - return False - - return True - - -def create_command_list(data_folder, combination_dict, extra_arg="", select=""): - if len(select) > 0: - for parameter in select.split(","): - parameter = parameter.strip() - key, value = parameter.split(":") - values = combination_dict[key] - for v in values: - if value in str(v): - combination_dict[key] = [v] - break - - combinations = list(itertools.product(*list(combination_dict.values()))) - command_list = [] - for c in combinations: - command_dict = dict(zip(list(combination_dict.keys()), c)) - - if not parameter_checkers(command_dict): - continue - - command_str = f'python main_analogvnn1.py --data_folder "{data_folder}"' - for key, value in command_dict.items(): - if value is None: - continue - if inspect.isclass(value): - command_str += f' --{key} "{value.__name__}"' - elif isinstance(value, str): - command_str += f' --{key} "{value}"' - else: - command_str += f' --{key} {value}' - - command_str += f" {extra_arg}" - command_list.append(command_str) - command_list = natsorted(command_list, alg=ns.IGNORECASE) - return command_list - - -def run_combination_1(data_folder, extra_arg="", select=""): - combination_dict = OrderedDict({ - "dataset": analogvnn1_parameters_list["dataset"], - - "num_conv_layer": analogvnn1_parameters_list["nn_model_params"]["num_conv_layer"], - "num_linear_layer": analogvnn1_parameters_list["nn_model_params"]["num_linear_layer"], - "activation_class": analogvnn1_parameters_list["nn_model_params"]["activation_class"], - "norm_class": analogvnn1_parameters_list["nn_model_params"]["norm_class"], - "approach": analogvnn1_parameters_list["nn_model_params"]["approach"], - - "w_norm_class": analogvnn1_parameters_list["weight_model_params"]["norm_class"], - }) - return create_command_list(data_folder, combination_dict, extra_arg=extra_arg, select=select) - - -def run_combination_2(data_folder, extra_arg="", select=""): - activation_class = copy.deepcopy(analogvnn1_parameters_list["nn_model_params"]["activation_class"]) - activation_class.remove(None) - precision_class = copy.deepcopy(analogvnn1_parameters_list["nn_model_params"]["precision_class"]) - precision_class.remove(None) - norm_class = [Clamp, L2Norm] - - combination_dict = OrderedDict({ - "dataset": analogvnn1_parameters_list["dataset"], - - "num_conv_layer": analogvnn1_parameters_list["nn_model_params"]["num_conv_layer"], - "num_linear_layer": analogvnn1_parameters_list["nn_model_params"]["num_linear_layer"], - "activation_class": activation_class, - "norm_class": norm_class, - "precision_class": precision_class, - "precision": analogvnn1_parameters_list["nn_model_params"]["precision"], - - "w_norm_class": norm_class, - "w_precision_class": precision_class, - "w_precision": analogvnn1_parameters_list["weight_model_params"]["precision"], - }) - return create_command_list(data_folder, combination_dict, extra_arg=extra_arg, select=select) - - -def run_combination_3(data_folder, extra_arg="", select=""): - activation_class = copy.deepcopy(analogvnn1_parameters_list["nn_model_params"]["activation_class"]) - activation_class.remove(None) - precision_class = copy.deepcopy(analogvnn1_parameters_list["nn_model_params"]["precision_class"]) - precision_class.remove(None) - norm_class = [Clamp] - noise_class = copy.deepcopy(analogvnn1_parameters_list["nn_model_params"]["noise_class"]) - noise_class.remove(None) - - combination_dict = OrderedDict({ - "dataset": analogvnn1_parameters_list["dataset"], - - "num_conv_layer": analogvnn1_parameters_list["nn_model_params"]["num_conv_layer"], - "num_linear_layer": analogvnn1_parameters_list["nn_model_params"]["num_linear_layer"], - "activation_class": activation_class, - "norm_class": norm_class, - "precision_class": precision_class, - "precision": analogvnn1_parameters_list["nn_model_params"]["precision"], - "noise_class": noise_class, - "leakage": analogvnn1_parameters_list["nn_model_params"]["leakage"], - - "w_norm_class": norm_class, - "w_precision_class": precision_class, - "w_precision": analogvnn1_parameters_list["weight_model_params"]["precision"], - "w_noise_class": noise_class, - "w_leakage": analogvnn1_parameters_list["weight_model_params"]["leakage"], - }) - return create_command_list(data_folder, combination_dict, extra_arg=extra_arg, select=select) - - -RUN_UNDER_12_LIST = { - "11lm": (run_combination_1, "num_linear_layer:1,num_conv_layer:0,approach:default,dataset:MNIST"), - "11cm": (run_combination_1, "num_linear_layer:1,num_conv_layer:3,approach:default,dataset:MNIST"), - "11lf": (run_combination_1, "num_linear_layer:1,num_conv_layer:0,approach:default,dataset:FashionMNIST"), - "11cf": (run_combination_1, "num_linear_layer:1,num_conv_layer:3,approach:default,dataset:FashionMNIST"), - "11lc": (run_combination_1, "num_linear_layer:1,num_conv_layer:0,approach:default,dataset:CIFAR10"), - "11cc": (run_combination_1, "num_linear_layer:1,num_conv_layer:3,approach:default,dataset:CIFAR10"), - "12lm": (run_combination_1, "num_linear_layer:2,num_conv_layer:0,approach:default,dataset:MNIST"), - "12cm": (run_combination_1, "num_linear_layer:2,num_conv_layer:3,approach:default,dataset:MNIST"), - "12lf": (run_combination_1, "num_linear_layer:2,num_conv_layer:0,approach:default,dataset:FashionMNIST"), - "12cf": (run_combination_1, "num_linear_layer:2,num_conv_layer:3,approach:default,dataset:FashionMNIST"), - "12lc": (run_combination_1, "num_linear_layer:2,num_conv_layer:0,approach:default,dataset:CIFAR10"), - "12cc": (run_combination_1, "num_linear_layer:2,num_conv_layer:3,approach:default,dataset:CIFAR10"), - "13lm": (run_combination_1, "num_linear_layer:3,num_conv_layer:0,approach:default,dataset:MNIST"), - "13cm": (run_combination_1, "num_linear_layer:3,num_conv_layer:3,approach:default,dataset:MNIST"), - "13lf": (run_combination_1, "num_linear_layer:3,num_conv_layer:0,approach:default,dataset:FashionMNIST"), - "13cf": (run_combination_1, "num_linear_layer:3,num_conv_layer:3,approach:default,dataset:FashionMNIST"), - "13lc": (run_combination_1, "num_linear_layer:3,num_conv_layer:0,approach:default,dataset:CIFAR10"), - "13cc": (run_combination_1, "num_linear_layer:3,num_conv_layer:3,approach:default,dataset:CIFAR10"), - - # "2lml": (run_combination_2, "num_conv_layer:0,dataset:MNIST,norm_class:L2Norm"), - # "2cml": (run_combination_2, "num_conv_layer:3,dataset:MNIST,norm_class:L2Norm"), - # "2lfl": (run_combination_2, "num_conv_layer:0,dataset:FashionMNIST,norm_class:L2Norm"), - # "2cfl": (run_combination_2, "num_conv_layer:3,dataset:FashionMNIST,norm_class:L2Norm"), - # "2lcl": (run_combination_2, "num_conv_layer:0,dataset:CIFAR10,norm_class:L2Norm"), - # "2ccl": (run_combination_2, "num_conv_layer:3,dataset:CIFAR10,norm_class:L2Norm"), - # "2lmc": (run_combination_2, "num_conv_layer:0,dataset:MNIST,norm_class:Clamp"), - # "2cmc": (run_combination_2, "num_conv_layer:3,dataset:MNIST,norm_class:Clamp"), - # "2lfc": (run_combination_2, "num_conv_layer:0,dataset:FashionMNIST,norm_class:Clamp"), - # "2cfc": (run_combination_2, "num_conv_layer:3,dataset:FashionMNIST,norm_class:Clamp"), - # "2lcc": (run_combination_2, "num_conv_layer:0,dataset:CIFAR10,norm_class:Clamp"), - # "2ccc": (run_combination_2, "num_conv_layer:3,dataset:CIFAR10,norm_class:Clamp"), - # - # "3glmr": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:0,dataset:MNIST,precision_class:ReducePrecision,w_precision_class:ReducePrecision" - # ), - # "3gcmr": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:3,dataset:MNIST,precision_class:ReducePrecision,w_precision_class:ReducePrecision" - # ), - # "3glfr": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:0,dataset:FashionMNIST,precision_class:ReducePrecision,w_precision_class:ReducePrecision" - # ), - # "3gcfr": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:3,dataset:FashionMNIST,precision_class:ReducePrecision,w_precision_class:ReducePrecision" - # ), - # "3glcr": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:0,dataset:CIFAR10,precision_class:ReducePrecision,w_precision_class:ReducePrecision" - # ), - # "3gccr": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:3,dataset:CIFAR10,precision_class:ReducePrecision,w_precision_class:ReducePrecision" - # ), - # - # "3glms": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:0,dataset:MNIST,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision" - # ), - # "3gcms": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:3,dataset:MNIST,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision" - # ), - # "3glfs": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:0,dataset:FashionMNIST,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision" - # ), - # "3gcfs": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:3,dataset:FashionMNIST,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision" - # ), - # "3glcs": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:0,dataset:CIFAR10,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision" - # ), - # "3gccs": ( - # run_combination_3, - # "noise_class:GaussianNoise,w_noise_class:GaussianNoise," - # "num_conv_layer:3,dataset:CIFAR10,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision" - # ), - - # "3lmpr": (run_combination_3, - # "num_conv_layer:0,dataset:MNIST,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:ReducePrecision,w_precision_class:ReducePrecision"), - # "3cmpr": (run_combination_3, - # "num_conv_layer:3,dataset:MNIST,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:ReducePrecision,w_precision_class:ReducePrecision"), - # "3lfpr": (run_combination_3, - # "num_conv_layer:0,dataset:FashionMNIST,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:ReducePrecision,w_precision_class:ReducePrecision"), - # "3cfpr": (run_combination_3, - # "num_conv_layer:3,dataset:FashionMNIST,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:ReducePrecision,w_precision_class:ReducePrecision"), - # "3lcpr": (run_combination_3, - # "num_conv_layer:0,dataset:CIFAR10,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:ReducePrecision,w_precision_class:ReducePrecision"), - # "3ccpr": (run_combination_3, - # "num_conv_layer:3,dataset:CIFAR10,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:ReducePrecision,w_precision_class:ReducePrecision"), - - # "3lmps": (run_combination_3, - # "num_conv_layer:0,dataset:MNIST,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision"), - # "3cmps": (run_combination_3, - # "num_conv_layer:3,dataset:MNIST,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision"), - # "3lfps": (run_combination_3, - # "num_conv_layer:0,dataset:FashionMNIST,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision"), - # "3cfps": (run_combination_3, - # "num_conv_layer:3,dataset:FashionMNIST,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision"), - # "3lcps": (run_combination_3, - # "num_conv_layer:0,dataset:CIFAR10,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision"), - # "3ccps": (run_combination_3, - # "num_conv_layer:3,dataset:CIFAR10,noise_class:PoissonNoise,w_noise_class:GaussianNoise,precision_class:StochasticReducePrecision,w_precision_class:StochasticReducePrecision"), -} - - -def run_combination_main(): - parser = argparse.ArgumentParser() - parser.add_argument("--memory_required", type=int, required=True) - parser.add_argument("--data_folder", type=str, required=True) - parser.add_argument("--select", type=str, default="") - parser.add_argument("--run_combination", type=str, required=True) - parser.add_argument("--run_index", type=int, default=-1) - parser.add_argument("--single_run", action='store_true') - parser.set_defaults(single_run=False) - all_arguments = parser.parse_known_args() - print(all_arguments) - - kwargs = vars(all_arguments[0]) - extra_arg = "" - for i in all_arguments[1]: - if " " in i: - extra_arg += f' "{i}"' - else: - extra_arg += f' {i}' - - if torch.cuda.is_available(): - num_process = max(math.floor(torch.cuda.mem_get_info()[1] / (kwargs['memory_required'] * 1024 ** 2)), 1) - else: - num_process = 1 - - print(f"memory_required: {kwargs['memory_required']}") - print(f"num_process: {num_process}") - print(f"data_folder: {kwargs['data_folder']}") - print(f"run_combination: {kwargs['run_combination']}") - print(f"run_index: {kwargs['run_index']}") - prepare_data_folder(kwargs['data_folder']) - - select = RUN_UNDER_12_LIST[kwargs['run_combination']][1] - if len(kwargs['select']) > 0: - select += "," + kwargs['select'] - - command_list = RUN_UNDER_12_LIST[kwargs['run_combination']][0](kwargs['data_folder'], extra_arg, select) - print(f"number of combinations: {len(command_list)}") - print() - print() - - command_list = [(kwargs['data_folder'], x) for x in command_list] - if kwargs['run_index'] >= 0: - command_list = [command_list[kwargs['run_index'] - 1]] - - if kwargs["single_run"]: - command_list = command_list[:kwargs["num_process"]] - - with ThreadPool(num_process) as pool: - pool.map(run_command, command_list) - - -def create_slurm_scripts(): - shutil.rmtree("_crc_slurm") - os.mkdir("_crc_slurm") - with open("run_array_@@@.slurm", "r") as main_run_file: - text = main_run_file.read() - - for i in RUN_UNDER_12_LIST: - with open(f"_crc_slurm/run_analogvnn1_{i}.slurm", "w") as file: - file.write( - text - .replace("@@@cpu@@@", "3") - .replace("@@@partition@@@", "scavenger") - .replace("@@@time@@@", "1-00:00:00") - .replace("@@@VideoMemoryRequired@@@", "11000") - .replace("@@@RunScript@@@", Path(__file__).name.split(".")[0]) - .replace("@@@run_combination@@@", i) - .replace("@@@array@@@", f"1-{len(RUN_UNDER_12_LIST[i][0]('', '', RUN_UNDER_12_LIST[i][1]))}") - .replace("@@@extra@@@", "") - ) - # file.write( - # text - # .replace("@@@cpu@@@", "16") - # .replace("@@@partition@@@", "a100") - # .replace("@@@time@@@", "2-00:00:00") - # .replace("@@@VideoMemoryRequired@@@", "1800") - # .replace("@@@RunScript@@@", Path(__file__).name.split(".")[0]) - # .replace("@@@run_combination@@@", i) - # ) - - with open(f"_crc_slurm/_run.sh", "w") as file: - for i in RUN_UNDER_12_LIST: - file.write(f"sbatch run_analogvnn1_{i}.slurm\n") - - -if __name__ == '__main__': - create_slurm_scripts() - for name, value in RUN_UNDER_12_LIST.items(): - size = len(value[0]('', '', value[1])) - print(f"{name}: {size}, {size * 0.01080530071}") - print() - # run_combination_main() diff --git a/research/parallel_failed_analogvnn1.py b/research/parallel_failed_analogvnn1.py deleted file mode 100644 index b3f3e38..0000000 --- a/research/parallel_failed_analogvnn1.py +++ /dev/null @@ -1,51 +0,0 @@ -import argparse -from multiprocessing.pool import ThreadPool -from pathlib import Path - -from parallel_analogvnn1 import run_command, prepare_data_folder - - -def run_failed_combinations(): - parser = argparse.ArgumentParser() - parser.add_argument("--num_process", type=int, required=True) - parser.add_argument("--data_folder", type=str, required=True) - parser.add_argument("--failed_file", type=str, default=True) - parser.add_argument("--single_run", action='store_true') - parser.set_defaults(single_run=False) - all_arguments = parser.parse_known_args() - print(all_arguments) - - kwargs = vars(all_arguments[0]) - extra_arg = "" - for i in all_arguments[1]: - if " " in i: - extra_arg += f' "{i}"' - else: - extra_arg += f' {i}' - - print(f"num_process: {kwargs['num_process']}") - print(f"data_folder: {kwargs['data_folder']}") - print(f"failed_file: {kwargs['failed_file']}") - prepare_data_folder(kwargs['data_folder']) - - command_list = [] - with open(Path(kwargs["failed_file"]), "r") as file: - for line in file.readlines(): - command_list.append(line.split("::")[-1]) - - print(f"number of runs: {len(command_list)}") - print() - print() - - command_list = [(kwargs['data_folder'], x) for x in command_list] - # print(command_list) - - if kwargs["single_run"]: - command_list = command_list[:kwargs["num_process"]] - - with ThreadPool(kwargs["num_process"]) as pool: - pool.map(run_command, command_list) - - -if __name__ == '__main__': - run_failed_combinations() diff --git a/research/parallel_parneet.py b/research/parallel_parneet.py deleted file mode 100644 index 752fc5e..0000000 --- a/research/parallel_parneet.py +++ /dev/null @@ -1,329 +0,0 @@ -import argparse -import copy -import hashlib -import inspect -import itertools -import math -import os -import shutil -import subprocess -import time -from collections import OrderedDict -from multiprocessing.pool import ThreadPool -from pathlib import Path - -import torch -import torchvision -from natsort import natsorted, ns - -from analogvnn.nn.activation.ELU import ELU -from analogvnn.nn.activation.Gaussian import GeLU -from analogvnn.nn.activation.ReLU import LeakyReLU, ReLU -from analogvnn.nn.activation.SiLU import SiLU -from analogvnn.nn.activation.Tanh import Tanh -from analogvnn.nn.noise.GaussianNoise import GaussianNoise -from analogvnn.nn.normalize.Clamp import * -from analogvnn.nn.normalize.LPNorm import L1Norm, L2Norm, L1NormW, L2NormW, L1NormM, L2NormM, L1NormWM, L2NormWM -from analogvnn.nn.precision.ReducePrecision import ReducePrecision -from analogvnn.nn.precision.StochasticReducePrecision import StochasticReducePrecision - -combination_dict = OrderedDict({ - "color": [False, True], - "activation_class": [ReLU, LeakyReLU, SiLU, ELU, GeLU, Tanh], - "norm_class": [None, Clamp, L1Norm, L2Norm, L1NormW, L2NormW, L1NormM, L2NormM, L1NormWM, L2NormWM], - "w_norm_class": [None, Clamp, L1Norm, L2Norm, L1NormW, L2NormW, L1NormM, L2NormM, L1NormWM, L2NormWM], - "batch_size": [128, 256, 384, 512], - "precision_class": [None, ReducePrecision, StochasticReducePrecision], - "noise_class": [None, GaussianNoise], - - "precision": [None, 4, 8, 16, 32, 64], - "w_precision": [None, 4, 8, 16, 32, 64], - "leakage": [None, 0.2, 0.4, 0.6, 0.8], - "w_leakage": [None, 0.2, 0.4, 0.6, 0.8], -}) - -RUN_PARNEET_LIST = { - # "all": "", - # b - # "nb": "norm_class:None,w_norm_class:None,precision_class:None", - # "cb": "norm_class:Clamp,w_norm_class:Clamp,precision_class:None", - - # n - "ng": "precision_class:None,batch_size:512,color:False", - "nc": "precision_class:None,batch_size:512,color:True", - - # p - # "lp": "activation_class:LeakyReLU,precision_class:~None,noise_class:None,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "gp": "activation_class:GeLU,precision_class:~None,noise_class:None,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "rp": "activation_class:ReLU,precision_class:~None,noise_class:None,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "sp": "activation_class:SiLU,precision_class:~None,noise_class:None,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "ep": "activation_class:ELU,precision_class:~None,noise_class:None,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "tp": "activation_class:Tanh,precision_class:~None,noise_class:None,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - - # g - # "lrg": "activation_class:LeakyReLU,precision_class:ReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "lsg": "activation_class:LeakyReLU,precision_class:StochasticReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "grg": "activation_class:GeLU,precision_class:ReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - "gsg": "activation_class:GeLU,precision_class:StochasticReducePrecision," - "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "rrg": "activation_class:ReLU,precision_class:ReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "rsg": "activation_class:ReLU,precision_class:StochasticReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "srg": "activation_class:SiLU,precision_class:ReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "ssg": "activation_class:SiLU,precision_class:StochasticReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "erg": "activation_class:ELU,precision_class:ReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "esg": "activation_class:ELU,precision_class:StochasticReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "trg": "activation_class:Tanh,precision_class:ReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", - # "tsg": "activation_class:Tanh,precision_class:StochasticReducePrecision," - # "noise_class:GaussianNoise,norm_class:Clamp,w_norm_class:Clamp,batch_size:512", -} - - -def prepare_data_folder(folder_path): - folder_path = Path(folder_path) - runtime_path = folder_path.joinpath("runtime") - datasets_path = folder_path.joinpath("datasets") - models_path = folder_path.joinpath("models") - tensorboard_path = folder_path.joinpath("tensorboard") - logs_path = folder_path.joinpath("logs") - - if not folder_path.exists(): - os.mkdir(folder_path) - if not runtime_path.exists(): - os.mkdir(runtime_path) - if not datasets_path.exists(): - os.mkdir(datasets_path) - if not models_path.exists(): - os.mkdir(models_path) - if not tensorboard_path.exists(): - os.mkdir(tensorboard_path) - if not logs_path.exists(): - os.mkdir(logs_path) - - torchvision.datasets.CIFAR10(root=str(datasets_path.absolute()), download=True) - - -def run_command(command): - data_folder, command = command - runtime = Path(data_folder).joinpath("runtime") - hash_id = hashlib.sha256(str(command).encode("utf-8")).hexdigest() - timestamp = f"{int(time.time() * 1000)}" - - command += f" --data_folder {data_folder}" - if "--timestamp" not in command: - command += f" --timestamp {timestamp}" - else: - timestamp = command.split("--timestamp")[-1] - timestamp = timestamp.strip().split(" ")[0] - - if "--name" not in command: - command += f" --name {hash_id}" - else: - hash_id = command.split("--name")[-1] - hash_id = hash_id.strip().split(" ")[0] - - filename = f"{timestamp}_{hash_id}" - out_file = runtime.joinpath(f"{filename}.log") - - with open(out_file, "w+") as out: - out.write(command + "\n") - out.write(f"Running {filename} :: {command}\n\n") - print(f"Running {filename} :: {command}") - - p = subprocess.Popen(command, shell=True, stdout=out, stderr=out) - p.wait() - rc = p.returncode - - out.write(f"\n\n") - if rc == 0: - out.write(f"Success {p.pid} :: {filename} :: {command}") - print(f"Success {p.pid} :: {filename} :: {command}") - else: - out.write(f"Failed {p.pid} :: {filename} :: {rc} :: {command}") - print(f"Failed {p.pid} :: {filename} :: {rc} :: {command}") - - out.write(f"\n\n{rc}") - - -def parameter_checkers(parameters): - if ("norm_class" not in parameters or parameters["norm_class"] is None) and "approach" in parameters: - if parameters["approach"] != "default": - return False - - return True - - -def create_command_list(extra_arg="", select=""): - cd_copy = copy.deepcopy(combination_dict) - if len(select) > 0: - for parameter in select.split(","): - parameter = parameter.strip() - key, value = parameter.split(":") - not_in = value[0] == "~" - if not_in: - value = value[1:] - - values = cd_copy[key] - cd_copy[key] = [] - for v in values: - if inspect.isclass(v): - v = v.__name__ - if value == str(v) and not not_in: - cd_copy[key].append(v) - if value != str(v) and not_in: - cd_copy[key].append(v) - - combinations = list(itertools.product(*list(cd_copy.values()))) - - command_list = [] - for c in combinations: - command_dict = dict(zip(list(cd_copy.keys()), c)) - # command_dict["w_norm_class"] = command_dict["norm_class"] - command_dict["w_precision_class"] = command_dict["precision_class"] - command_dict["w_noise_class"] = command_dict["noise_class"] - - if command_dict["norm_class"] is None and \ - (command_dict["precision_class"] is not None or command_dict["noise_class"] is not None): - continue - - if command_dict["precision_class"] is None and command_dict["precision"] is not None: - continue - if command_dict["noise_class"] is None and command_dict["leakage"] is not None: - continue - if command_dict["w_precision_class"] is None and command_dict["w_precision"] is not None: - continue - if command_dict["w_noise_class"] is None and command_dict["w_leakage"] is not None: - continue - if command_dict["precision_class"] is not None and command_dict["precision"] is None: - continue - if command_dict["noise_class"] is not None and command_dict["leakage"] is None: - continue - if command_dict["w_precision_class"] is not None and command_dict["w_precision"] is None: - continue - if command_dict["w_noise_class"] is not None and command_dict["w_leakage"] is None: - continue - - if command_dict["noise_class"] is not None and command_dict["precision"] is None: - continue - if command_dict["w_noise_class"] is not None and command_dict["w_precision"] is None: - continue - - if not parameter_checkers(command_dict): - continue - - command_str = f'python main_parneet.py' - for key, value in command_dict.items(): - if value is None: - continue - if inspect.isclass(value): - command_str += f' --{key} "{value.__name__}"' - elif isinstance(value, str): - command_str += f' --{key} "{value}"' - else: - command_str += f' --{key} {value}' - - command_str += f" {extra_arg}" - command_list.append(command_str.strip()) - # print(command_str) - command_list = natsorted(command_list, alg=ns.IGNORECASE) - # with open(f"_data/{list(RUN_PARNEET_LIST.keys())[list(RUN_PARNEET_LIST.values()).index(select)]}.txt", "w") as file: - # file.write("\n".join(command_list)) - return command_list - - -def run_combination_main(): - parser = argparse.ArgumentParser() - parser.add_argument("--memory_required", type=int, required=True) - parser.add_argument("--data_folder", type=str, required=True) - parser.add_argument("--select", type=str, default="") - parser.add_argument("--run_combination", type=str, required=True) - parser.add_argument("--run_index", type=int, default=-1) - parser.add_argument("--single_run", action='store_true') - parser.set_defaults(single_run=False) - all_arguments = parser.parse_known_args() - print(all_arguments) - - kwargs = vars(all_arguments[0]) - extra_arg = "" - for i in all_arguments[1]: - if " " in i: - extra_arg += f' "{i}"' - else: - - extra_arg += f' {i}' - - if torch.cuda.is_available(): - num_process = max(math.floor(torch.cuda.mem_get_info()[1] / (kwargs['memory_required'] * 1024 ** 2)), 1) - else: - num_process = 1 - - print(f"memory_required: {kwargs['memory_required']}") - print(f"num_process: {num_process}") - print(f"data_folder: {kwargs['data_folder']}") - print(f"run_combination: {kwargs['run_combination']}") - print(f"run_index: {kwargs['run_index']}") - prepare_data_folder(kwargs['data_folder']) - - select = RUN_PARNEET_LIST[kwargs['run_combination']] - if len(kwargs['select']) > 0: - select += "," + kwargs['select'] - - command_list = create_command_list(extra_arg, select) - print(f"number of combinations: {len(command_list)}") - print() - print() - - command_list = [(kwargs['data_folder'], x) for x in command_list] - if kwargs['run_index'] >= 0: - command_list = [command_list[kwargs['run_index'] - 1]] - - if kwargs["single_run"]: - command_list = command_list[:num_process] - - with ThreadPool(num_process) as pool: - pool.map(run_command, command_list) - - -def create_slurm_scripts(): - shutil.rmtree("_crc_slurm") - os.mkdir("_crc_slurm") - with open("run_array_@@@.slurm", "r") as main_run_file: - text = main_run_file.read() - - with open(f"_crc_slurm/_run.sh", "w") as run_file: - for i in RUN_PARNEET_LIST: - with open(f"_crc_slurm/run_parneet_{i}.slurm", "w") as slurm_file: - slurm_file.write( - text - .replace("@@@cpu@@@", "2") - .replace("@@@partition@@@", "scavenger") - .replace("@@@time@@@", "1-00:00:00") - .replace("@@@VideoMemoryRequired@@@", "11000") - .replace("@@@RunScript@@@", Path(__file__).name.split(".")[0]) - .replace("@@@run_combination@@@", i) - .replace("@@@array@@@", f"1-{len(create_command_list('', RUN_PARNEET_LIST[i]))}") - .replace("@@@extra@@@", "") - ) - - run_file.write(f"sbatch run_parneet_{i}.slurm\n") - - -if __name__ == '__main__': - create_slurm_scripts() - total = 0 - for name, value in RUN_PARNEET_LIST.items(): - size = len(create_command_list('', value)) - total += size - print(f"{name}: {size}, {size * 0.2197231834}") - print(f"total: {total}") - - # run_combination_main() diff --git a/research/run_@@@.slurm b/research/run_@@@.slurm deleted file mode 100644 index 777ea68..0000000 --- a/research/run_@@@.slurm +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=run_@@@RunScript@@@_@@@run_combination@@@ -#SBATCH --output=slurm_%x_%A.out -#SBATCH --mail-user=vis77@pitt.edu -#SBATCH --mail-type=ALL -#SBATCH --nodes=1 -#SBATCH --tasks-per-node=1 -#SBATCH --cpus-per-task=@@@cpu@@@ -#SBATCH --cluster=gpu -#SBATCH --partition=@@@partition@@@ -#SBATCH --gres=gpu:1 -#SBATCH --time=@@@time@@@ -#SBATCH --chdir="/ihome/nyoungblood/vis77" -#SBATCH --requeue -@@@extra@@@ - -source ~/.bashrc - -RunDirectoryLocation=$HOME/Vivswan-AnalogVNN -RunScript=@@@RunScript@@@.py -CondaEnv="${HOME}"/storage/envs/AnalogVNN_"${CPU_ARCHITECTURE}"_3.7 - -StorageDirectory="${HOME}/storage/" - - -cd ~ || exit -conda activate $CondaEnv - -echo "" -echo "####################################### nvidia-smi #######################################" -echo "" -nvidia-smi -echo "" -echo "" - -echo "" -echo "####################################### lsb_release #######################################" -echo "" -/usr/bin/lsb_release -a -echo "" -echo "" - -echo "" -echo "####################################### printenv #######################################" -echo "" -printenv -echo "" -echo "" - -echo "####################################### Conda Environment #######################################" -echo "" -echo "" -conda list -echo "" -echo "" - -SlurmScratchName="slurm_${SLURM_JOB_NAME}_${SLURM_JOBID}" -SlurmScratchDirectory="${SLURM_SCRATCH}/${SlurmScratchName}" -mkdir -p "${SlurmScratchDirectory}" -mkdir -p "${SlurmScratchDirectory}"/_results -mkdir -p "${SlurmScratchDirectory}"/_results/datasets/ -mkdir -p "${SlurmScratchDirectory}"/_results/runtime/ -mkdir -p "${SlurmScratchDirectory}"/_results/models/ -mkdir -p "${SlurmScratchDirectory}"/_results/tensorboard/ -mkdir -p "${SlurmScratchDirectory}"/_results/logs/ -rsync -ar "${HOME}"/storage/_datasets/ "${SlurmScratchDirectory}"/_results/datasets/ -rsync -ar "${RunDirectoryLocation}"/ "${SlurmScratchDirectory}"/ - -run_on_exit(){ - echo "####################################### Exit Began #######################################" - rm -rf "${SlurmScratchDirectory}/_results/datasets/" - cd "${SLURM_SCRATCH}" || exit - tar -czf "${SlurmScratchName}.tar.gz" "${SlurmScratchName}" - mv "${SlurmScratchName}.tar.gz" "${StorageDirectory}" - - echo "" - echo "####################################### Billing #######################################" - echo "" - sacct -M gpu -j $SLURM_JOBID --format=AllocTRES%50,elapsed - echo "" - - echo "" - echo "####################################### crc-job-stats.py #######################################" - echo "" - crc-job-stats.py - echo "" - echo "!!!!!!Completed!!!!!!!" - echo "" -} -trap run_on_exit EXIT - -cd "${SlurmScratchDirectory}" || exit - -echo "" -echo "####################################### Variables #######################################" -echo "" -echo "CPU_ARCHITECTURE = ${CPU_ARCHITECTURE}" -echo "SlurmScratchName = ${SlurmScratchName}" -echo "StorageDirectory = ${StorageDirectory}" -echo "SlurmScratchDirectory = ${SlurmScratchDirectory}" -echo "RunDirectoryLocation = ${RunDirectoryLocation}" -echo "RunScript = ${RunScript}" -echo "Array ID = ${SLURM_ARRAY_TASK_ID}" -echo "CondaEnv = ${CondaEnv}" -echo "which conda = $( which conda )" -echo "which python = $( which python )" -echo "which python3 = $( which python3 )" -echo "pytorch version" = $( python3 -c "import torch; print(torch.__version__)" ) -echo "tensorflow version" = $( python3 -c 'import tensorflow as tf; print(tf.__version__)' ) -echo "tensorboard version" = $( python3 -c 'from tensorboard import version; print(version.VERSION)' ) -echo "" -echo "" - -echo "" -echo "####################################### Main Program: Starting #######################################" -echo "" - -# time \ -srun python3 $RunScript \ ---memory_required @@@VideoMemoryRequired@@@ \ ---data_folder ./_results \ ---run_combination @@@run_combination@@@ \ ---cuda_memory @@@cuda_memory@@@ \ ---tensorboard \ ---save_data \ -#--single_run -#exit 125 - - -echo "" -echo "####################################### Main Program: Finished #######################################" -echo "" - diff --git a/research/run_array_@@@.slurm b/research/run_array_@@@.slurm deleted file mode 100644 index cae6b41..0000000 --- a/research/run_array_@@@.slurm +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=run_@@@RunScript@@@_@@@run_combination@@@ -#SBATCH --output=slurm_%x_%A_%a.out -#SBATCH --mail-user=vis77@pitt.edu -#SBATCH --mail-type=ALL -#SBATCH --tasks-per-node=1 -#SBATCH --cpus-per-task=@@@cpu@@@ -#SBATCH --cluster=gpu -#SBATCH --partition=@@@partition@@@ -#SBATCH --gres=gpu:1 -#SBATCH --time=@@@time@@@ -#SBATCH --chdir="/ihome/nyoungblood/vis77" -#SBATCH --array=@@@array@@@ -#SBATCH --requeue -#SBATCH --constraint=v100|titanx|gtx1080 -@@@extra@@@ - -source ~/.bashrc - -RunDirectoryLocation=$HOME/Vivswan-AnalogVNN -RunScript=@@@RunScript@@@.py -CondaEnv="${HOME}"/storage/envs/AnalogVNN_"${CPU_ARCHITECTURE}"_3.7 - -StorageDirectory="${HOME}/storage/" - - -cd ~ || exit -conda activate $CondaEnv - -echo "" -echo "####################################### nvidia-smi #######################################" -echo "" -nvidia-smi -echo "" -echo "" - -echo "" -echo "####################################### lsb_release #######################################" -echo "" -/usr/bin/lsb_release -a -echo "" -echo "" - -echo "" -echo "####################################### printenv #######################################" -echo "" -printenv -echo "" -echo "" - -echo "####################################### Conda Environment #######################################" -echo "" -echo "" -conda list -echo "" -echo "" - -SlurmScratchName="slurm_${SLURM_JOB_NAME}_${SLURM_ARRAY_JOB_ID}_${SLURM_ARRAY_TASK_ID}" -SlurmScratchDirectory="${SLURM_SCRATCH}/${SlurmScratchName}" -mkdir -p "${SlurmScratchDirectory}" -mkdir -p "${SlurmScratchDirectory}"/_results -mkdir -p "${SlurmScratchDirectory}"/_results/datasets/ -mkdir -p "${SlurmScratchDirectory}"/_results/runtime/ -mkdir -p "${SlurmScratchDirectory}"/_results/models/ -mkdir -p "${SlurmScratchDirectory}"/_results/tensorboard/ -mkdir -p "${SlurmScratchDirectory}"/_results/logs/ -rsync -ar "${HOME}"/storage/_datasets/ "${SlurmScratchDirectory}"/_results/datasets/ -rsync -ar "${RunDirectoryLocation}"/ "${SlurmScratchDirectory}"/ - -run_on_exit(){ - echo "####################################### Exit Began #######################################" - rm -rf "${SlurmScratchDirectory}/_results/datasets/" - cd "${SLURM_SCRATCH}" || exit - srun tar -czf "${SlurmScratchName}.tar.gz" "${SlurmScratchName}" - srun mv "${SlurmScratchName}.tar.gz" "${StorageDirectory}" - - echo "" - echo "####################################### Billing #######################################" - echo "" - sacct -M gpu -j $SLURM_JOBID --format=AllocTRES%50,elapsed - echo "" - - echo "" - echo "####################################### crc-job-stats.py #######################################" - echo "" - crc-job-stats.py - echo "" - echo "!!!!!!Completed!!!!!!!" - echo "" -} -trap run_on_exit EXIT - -cd "${SlurmScratchDirectory}" || exit - -echo "" -echo "####################################### Variables #######################################" -echo "" -echo "CPU_ARCHITECTURE = ${CPU_ARCHITECTURE}" -echo "SlurmScratchName = ${SlurmScratchName}" -echo "StorageDirectory = ${StorageDirectory}" -echo "SlurmScratchDirectory = ${SlurmScratchDirectory}" -echo "RunDirectoryLocation = ${RunDirectoryLocation}" -echo "RunScript = ${RunScript}" -echo "Array ID = ${SLURM_ARRAY_TASK_ID}" -echo "CondaEnv = ${CondaEnv}" -echo "which conda = $( which conda )" -echo "which python = $( which python )" -echo "which python3 = $( which python3 )" -echo "pytorch version" = $( python3 -c "import torch; print(torch.__version__)" ) -echo "tensorflow version" = $( python3 -c 'import tensorflow as tf; print(tf.__version__)' ) -echo "tensorboard version" = $( python3 -c 'from tensorboard import version; print(version.VERSION)' ) -echo "" -echo "" - -echo "" -echo "####################################### Main Program: Starting #######################################" -echo "" - -# time \ -srun python3 $RunScript \ ---memory_required @@@VideoMemoryRequired@@@ \ ---data_folder ./_results \ ---run_combination @@@run_combination@@@ \ ---run_index ${SLURM_ARRAY_TASK_ID} \ ---tensorboard \ ---save_data \ -#--single_run -#exit 125 - - -echo "" -echo "####################################### Main Program: Finished #######################################" -echo "" - diff --git a/research/utils/__init__.py b/research/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/research/utils/data_dirs.py b/research/utils/data_dirs.py deleted file mode 100644 index 5ead4d8..0000000 --- a/research/utils/data_dirs.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import shutil -import time -from dataclasses import dataclass - -from research.utils.path_functions import path_join - - -@dataclass -class DataPaths: - name: str - model_data: str - tensorboard: str - dataset: str - logs: str - timestamp: str - - -def data_dirs(path, name=None, timestamp=None, tensorboard=True): - if timestamp is None: - timestamp = str(int(time.time())) - - name = timestamp + ("" if name is None else ("_" + name)) - - dataset_path = path_join(path, "datasets") - logs_path = path_join(path, "logs") - - if not os.path.exists(path): - os.mkdir(path) - if not os.path.exists(dataset_path): - os.mkdir(dataset_path) - if not os.path.exists(logs_path): - os.mkdir(logs_path) - - if not os.path.exists(path_join(path, "models")): - os.mkdir(path_join(path, "models")) - if not os.path.exists(path_join(path, "tensorboard")): - os.mkdir(path_join(path, "tensorboard")) - - models_path = path_join(path, f"models/{name}") - tensorboard_path = path_join(path, f"tensorboard/{name}") - - if tensorboard: - os.mkdir(tensorboard_path) - - if models_path: - os.mkdir(models_path) - - return DataPaths( - name=name, - model_data=models_path, - tensorboard=tensorboard_path, - dataset=dataset_path, - logs=logs_path, - timestamp=timestamp, - ) - - -def erase_data_dirs(path): - if os.path.exists(path): - if os.path.exists(path_join(path, "models")): - shutil.rmtree(path_join(path, "models")) - if os.path.exists(path_join(path, "tensorboard")): - shutil.rmtree(path_join(path, "tensorboard")) diff --git a/research/utils/dataset_to_image.py b/research/utils/dataset_to_image.py deleted file mode 100644 index dc25a15..0000000 --- a/research/utils/dataset_to_image.py +++ /dev/null @@ -1,20 +0,0 @@ -import math - -import numpy as np -from PIL import Image - - -def to_image(element): - image_array = [] - batch = math.floor(math.sqrt(element.size()[0])) - element = np.array(element.tolist()) - for i in range(1, batch): - sub_array = [] - for j in range(1, batch): - sub_array.append(np.uint8(element[i * batch + j] * 255)) - image_array.append(sub_array) - image = np.block(image_array).T - if image.shape[-1] == 1: - image = image.T[0] - im = Image.fromarray(image) - return im diff --git a/research/utils/helper_functions.py b/research/utils/helper_functions.py deleted file mode 100644 index 005bc3c..0000000 --- a/research/utils/helper_functions.py +++ /dev/null @@ -1,10 +0,0 @@ -def pick_instanceof(arr: list, superclass): - result = [] - if superclass is None: - return result - - for i in arr: - if isinstance(i, superclass): - result.append(i) - - return result diff --git a/research/utils/path_functions.py b/research/utils/path_functions.py deleted file mode 100644 index 37f3eef..0000000 --- a/research/utils/path_functions.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -import pathlib - - -def get_relative_path(this, path): - return os.path.normpath(os.path.abspath(os.path.join( - pathlib.Path(this).parent.absolute(), path) - )) - - -def path_join(*args): - args = list(args) - if len(args) == 0: - return None - elif len(args) == 1: - return args[0] - else: - path = args.pop(0) - for i in args: - path = os.path.normpath(os.path.abspath(os.path.join(path, i))) - return path diff --git a/sample_code.py b/sample_code.py index a77defb..020437d 100644 --- a/sample_code.py +++ b/sample_code.py @@ -28,7 +28,6 @@ def load_vision_dataset(dataset, path, batch_size, is_cuda=False, grayscale=True Returns: A tuple containing the train and test data loaders, the input shape, and a tuple of class labels. """ - dataset_kwargs = { 'batch_size': batch_size, 'shuffle': True @@ -61,7 +60,7 @@ def load_vision_dataset(dataset, path, batch_size, is_cuda=False, grayscale=True def cross_entropy_accuracy(output, target) -> float: - """ Cross Entropy accuracy function + """Cross Entropy accuracy function. Args: output (torch.Tensor): output of the model from passing inputs @@ -77,7 +76,7 @@ def cross_entropy_accuracy(output, target) -> float: class LinearModel(FullSequential): def __init__(self, activation_class, norm_class, precision_class, precision, noise_class, leakage): - """Initialise LinearModel with 3 Dense layers + """Initialise LinearModel with 3 Dense layers. Args: activation_class: Activation Class @@ -108,7 +107,7 @@ def __init__(self, activation_class, norm_class, precision_class, precision, noi self.add_sequence(*self.all_layers) def add_layer(self, layer): - """ To add the analog layer + """To add the analog layer. Args: layer (BaseLayer): digital layer module @@ -126,7 +125,7 @@ def add_layer(self, layer): class WeightModel(FullSequential): def __init__(self, norm_class, precision_class, precision, noise_class, leakage): - """Initialize the WeightModel + """Initialize the WeightModel. Args: norm_class: Normalization Class @@ -150,39 +149,38 @@ def __init__(self, norm_class, precision_class, precision, noise_class, leakage) def run_linear3_model(): - """ The main function to train photonics image classifier with 3 linear/dense nn for MNIST dataset - """ + """The main function to train photonics image classifier with 3 linear/dense nn for MNIST dataset.""" torch.backends.cudnn.benchmark = True torch.manual_seed(0) device, is_cuda = is_cpu_cuda.is_using_cuda - print(f"Device: {device}") + print(f'Device: {device}') print() # Loading Data - print(f"Loading Data...") + print('Loading Data...') train_loader, test_loader, input_shape, classes = load_vision_dataset( dataset=torchvision.datasets.MNIST, - path="_data/", + path='_data/', batch_size=128, is_cuda=is_cuda ) # Creating Models - print(f"Creating Models...") + print('Creating Models...') nn_model = LinearModel( activation_class=GeLU, norm_class=Clamp, precision_class=ReducePrecision, precision=2 ** 4, noise_class=GaussianNoise, - leakage=0.2 + leakage=0.5 ) weight_model = WeightModel( norm_class=Clamp, precision_class=ReducePrecision, precision=2 ** 4, noise_class=GaussianNoise, - leakage=0.2 + leakage=0.5 ) # Setting Model Parameters @@ -195,7 +193,7 @@ def run_linear3_model(): nn_model.optimizer = optim.Adam(params=nn_model.parameters()) # Training - print(f"Starting Training...") + print('Starting Training...') for epoch in range(10): train_loss, train_accuracy = nn_model.train_on(train_loader, epoch=epoch) test_loss, test_accuracy = nn_model.test_on(test_loader, epoch=epoch) @@ -207,7 +205,7 @@ def run_linear3_model(): f' Testing Loss: {test_loss:.4f},' \ f' Testing Accuracy: {100. * test_accuracy:.0f}%\n' print(print_str) - print("Run Completed Successfully...") + print('Run Completed Successfully...') if __name__ == '__main__': diff --git a/research/sample_code_with_logs.py b/sample_code_with_logs.py similarity index 73% rename from research/sample_code_with_logs.py rename to sample_code_with_logs.py index 35b8163..ea64f2b 100644 --- a/research/sample_code_with_logs.py +++ b/sample_code_with_logs.py @@ -1,5 +1,4 @@ import json -import time from pathlib import Path import numpy as np @@ -21,8 +20,7 @@ def load_vision_dataset(dataset, path, batch_size, is_cuda=False, grayscale=True): - """ - Loads a vision dataset with optional grayscale conversion and CUDA support. + """Loads a vision dataset with optional grayscale conversion and CUDA support. Args: dataset (Type[torchvision.datasetsVisionDataset]): the dataset class to use (e.g. torchvision.datasets.MNIST) @@ -34,7 +32,6 @@ def load_vision_dataset(dataset, path, batch_size, is_cuda=False, grayscale=True Returns: A tuple containing the train and test data loaders, the input shape, and a tuple of class labels. """ - dataset_kwargs = { 'batch_size': batch_size, 'shuffle': True @@ -67,7 +64,7 @@ def load_vision_dataset(dataset, path, batch_size, is_cuda=False, grayscale=True def cross_entropy_accuracy(output, target) -> float: - """ Cross Entropy accuracy function + """Cross Entropy accuracy function. Args: output (torch.Tensor): output of the model from passing inputs @@ -83,7 +80,7 @@ def cross_entropy_accuracy(output, target) -> float: class LinearModel(FullSequential): def __init__(self, activation_class, norm_class, precision_class, precision, noise_class, leakage): - """ Linear Model with 3 dense nn + """Linear Model with 3 dense neural network layers. Args: activation_class: Activation Class @@ -114,7 +111,7 @@ def __init__(self, activation_class, norm_class, precision_class, precision, noi self.add_sequence(*self.all_layers) def add_layer(self, layer): - """ To add the analog layer + """To add the analog layer. Args: layer (BaseLayer): digital layer module @@ -132,7 +129,7 @@ def add_layer(self, layer): class WeightModel(FullSequential): def __init__(self, norm_class, precision_class, precision, noise_class, leakage): - """ Weight Model + """Weight Model. Args: norm_class: Normalization Class @@ -156,44 +153,42 @@ def __init__(self, norm_class, precision_class, precision, noise_class, leakage) def run_linear3_model(): - """ The main function to train photonics image classifier with 3 linear/dense nn for MNIST dataset - """ + """The main function to train photonics image classifier with 3 linear/dense nn for MNIST dataset.""" torch.backends.cudnn.benchmark = True torch.manual_seed(0) - data_path = Path("C:/X/").joinpath(str(int(time.time()))) - data_path = Path("C:/X/").joinpath("hi") + data_path = Path('_data') if not data_path.exists(): data_path.mkdir() device, is_cuda = is_cpu_cuda.is_using_cuda - print(f"Device: {device}") + print(f'Device: {device}') print() # Loading Data - print(f"Loading Data...") + print('Loading Data...') train_loader, test_loader, input_shape, classes = load_vision_dataset( dataset=torchvision.datasets.MNIST, - path="C:/X/_data/", + path=str(data_path), batch_size=128, is_cuda=is_cuda ) # Creating Models - print(f"Creating Models...") + print('Creating Models...') nn_model = LinearModel( activation_class=GeLU, norm_class=Clamp, precision_class=ReducePrecision, precision=2 ** 4, noise_class=GaussianNoise, - leakage=0.2 + leakage=0.5 ) weight_model = WeightModel( norm_class=Clamp, precision_class=ReducePrecision, precision=2 ** 4, noise_class=GaussianNoise, - leakage=0.2 + leakage=0.5 ) # Setting Model Parameters @@ -205,21 +200,21 @@ def run_linear3_model(): PseudoParameter.parametrize_module(nn_model, transformation=weight_model) nn_model.optimizer = optim.Adam(params=nn_model.parameters()) - nn_model.create_tensorboard(str(data_path.joinpath("tensorboard"))) + nn_model.create_tensorboard(str(data_path.joinpath('tensorboard'))) - print(f"Saving Summary and Graphs...") + print('Saving Summary and Graphs...') _, nn_model_summary = nn_model.tensorboard.add_summary(train_loader) _, weight_model_summary = nn_model.tensorboard.add_summary(train_loader, model=weight_model) - save_autograd_graph_from_module(data_path.joinpath(f"nn_model"), nn_model, next(iter(train_loader))[0]) - save_autograd_graph_from_module(data_path.joinpath(f"weight_model"), weight_model, torch.ones((1, 1))) + save_autograd_graph_from_module(data_path.joinpath('nn_model'), nn_model, next(iter(train_loader))[0]) + save_autograd_graph_from_module(data_path.joinpath('weight_model'), weight_model, torch.ones((1, 1))) save_autograd_graph_from_module( - data_path.joinpath(f"nn_model_autograd"), + data_path.joinpath('nn_model_autograd'), nn_model, next(iter(train_loader))[0], from_forward=True ) save_autograd_graph_from_module( - data_path.joinpath(f"weight_model_autograd"), + data_path.joinpath('weight_model_autograd'), weight_model, torch.ones((1, 1)), from_forward=True @@ -227,31 +222,31 @@ def run_linear3_model(): nn_model.tensorboard.add_graph(train_loader) nn_model.tensorboard.add_graph(train_loader, model=weight_model) - print(f"Creating Log File...") - with open(data_path.joinpath("logfile.log"), "a+", encoding="utf-8") as file: - file.write(str(nn_model.optimizer) + "\n\n") + print('Creating Log File...') + with open(data_path.joinpath('logfile.log'), 'a+', encoding='utf-8') as file: + file.write(str(nn_model.optimizer) + '\n\n') - file.write(str(nn_model) + "\n\n") - file.write(str(weight_model) + "\n\n") - file.write(f"{nn_model_summary}\n\n") - file.write(f"{weight_model_summary}\n\n") + file.write(str(nn_model) + '\n\n') + file.write(str(weight_model) + '\n\n') + file.write(f'{nn_model_summary}\n\n') + file.write(f'{weight_model_summary}\n\n') loss_accuracy = { - "train_loss": [], - "train_accuracy": [], - "test_loss": [], - "test_accuracy": [], + 'train_loss': [], + 'train_accuracy': [], + 'test_loss': [], + 'test_accuracy': [], } # Training - print(f"Starting Training...") - for epoch in range(1): + print('Starting Training...') + for epoch in range(10): train_loss, train_accuracy = nn_model.train_on(train_loader, epoch=epoch) test_loss, test_accuracy = nn_model.test_on(test_loader, epoch=epoch) - loss_accuracy["train_loss"].append(train_loss) - loss_accuracy["train_accuracy"].append(train_accuracy) - loss_accuracy["test_loss"].append(test_loss) - loss_accuracy["test_accuracy"].append(test_accuracy) + loss_accuracy['train_loss'].append(train_loss) + loss_accuracy['train_accuracy'].append(train_accuracy) + loss_accuracy['test_loss'].append(test_loss) + loss_accuracy['test_accuracy'].append(test_accuracy) str_epoch = str(epoch + 1).zfill(1) print_str = f'({str_epoch})' \ @@ -261,40 +256,40 @@ def run_linear3_model(): f' Testing Accuracy: {100. * test_accuracy:.0f}%\n' print(print_str) - with open(data_path.joinpath("logfile.log"), "a+") as file: + with open(data_path.joinpath('logfile.log'), 'a+') as file: file.write(print_str) - print("Run Completed Successfully...") + print('Run Completed Successfully...') metric_dict = { - "train_loss": loss_accuracy["train_loss"][-1], - "train_accuracy": loss_accuracy["train_accuracy"][-1], - "test_loss": loss_accuracy["test_loss"][-1], - "test_accuracy": loss_accuracy["test_accuracy"][-1], - "min_train_loss": np.min(loss_accuracy["train_loss"]), - "max_train_accuracy": np.max(loss_accuracy["train_accuracy"]), - "min_test_loss": np.min(loss_accuracy["test_loss"]), - "max_test_accuracy": np.max(loss_accuracy["test_accuracy"]), + 'train_loss': loss_accuracy['train_loss'][-1], + 'train_accuracy': loss_accuracy['train_accuracy'][-1], + 'test_loss': loss_accuracy['test_loss'][-1], + 'test_accuracy': loss_accuracy['test_accuracy'][-1], + 'min_train_loss': np.min(loss_accuracy['train_loss']), + 'max_train_accuracy': np.max(loss_accuracy['train_accuracy']), + 'min_test_loss': np.min(loss_accuracy['test_loss']), + 'max_test_accuracy': np.max(loss_accuracy['test_accuracy']), } nn_model.tensorboard.tensorboard.add_hparams( metric_dict=metric_dict, hparam_dict={ - "precision": 2 ** 4, - "leakage": 0.2, - "noise": "GaussianNoise", - "activation": "GeLU", - "precision_class": "ReducePrecision", - "norm_class": "Clamp", - "optimizer": "Adam", - "loss_function": "CrossEntropyLoss", - "accuracy_function": "cross_entropy_accuracy", - "batch_size": 128, - "epochs": 1, + 'precision': 2 ** 4, + 'leakage': 0.5, + 'noise': 'GaussianNoise', + 'activation': 'GeLU', + 'precision_class': 'ReducePrecision', + 'norm_class': 'Clamp', + 'optimizer': 'Adam', + 'loss_function': 'CrossEntropyLoss', + 'accuracy_function': 'cross_entropy_accuracy', + 'batch_size': 128, + 'epochs': 1, } ) - with open(data_path.joinpath("logfile.log"), "a+") as file: - file.write(json.dumps(metric_dict, sort_keys=True) + "\n\n") - file.write("Run Completed Successfully...") + with open(data_path.joinpath('logfile.log'), 'a+') as file: + file.write(json.dumps(metric_dict, sort_keys=True) + '\n\n') + file.write('Run Completed Successfully...') nn_model.tensorboard.close() print() diff --git a/unit_tests/test_model_graphs.py b/unit_tests/test_model_graphs.py index fdde2e5..1d0d115 100644 --- a/unit_tests/test_model_graphs.py +++ b/unit_tests/test_model_graphs.py @@ -4,7 +4,7 @@ from torch import nn from analogvnn.graph.ModelGraph import ModelGraph -from analogvnn.utils.render_autograd_graph import make_autograd_do +from analogvnn.utils.render_autograd_graph import get_autograd_dot_from_outputs if __name__ == '__main__': mg = ModelGraph() @@ -12,19 +12,15 @@ l1 = nn.Linear(1, 1, bias=False) l1.weight.data = torch.ones_like(l1.weight.data) * 2 - def l2(*x): return torch.add(*x), torch.sub(*x) - def l3(x, y): return {"a": torch.sub(x, y), "b": torch.add(x, y)} - def l4(x, y, z, a, b): return ((x + y) + (a + b)) + z - def l5(x): return {"c": x * 0.5} @@ -69,7 +65,7 @@ def l5(x): mg.use_autograd_graph = True inputs = torch.ones((1, 1), dtype=torch.float, requires_grad=True) output = mg.forward_graph.calculate(inputs) - make_autograd_do(output, params={ + get_autograd_dot_from_outputs(output, named_params={ "input": inputs, "output": output, "l1.weight": l1.weight, diff --git a/unit_tests/test_pseudo_parameter.py b/unit_tests/test_pseudo_parameter.py index 726d5ef..6291243 100644 --- a/unit_tests/test_pseudo_parameter.py +++ b/unit_tests/test_pseudo_parameter.py @@ -43,16 +43,13 @@ def __init__(self): def forward(self, x): return x + (torch.ones_like(x) * self.weight) - class Symmetric(BackwardIdentity, Model): def forward(self, x): return torch.rand((1, x.size()[0])) @ x @ torch.rand((x.size()[1], 1)) - def pstr(s): return str(s).replace(" ", "").replace("\n", "") - model = Layer() parametrization = Symmetric() # parametrization.eval()