From 630afef0d1dd0fb4cab94bb2760d48af4fc2f243 Mon Sep 17 00:00:00 2001 From: Infinitode Date: Sun, 22 Sep 2024 20:48:01 +0200 Subject: [PATCH] Initial commit. --- .github/workflows/build_and_publish.yml | 34 +++ .gitignore | 5 + LICENSE | 7 + README.md | 85 ++++++ funcprofiler/__init__.py | 336 ++++++++++++++++++++++++ setup.py | 26 ++ test.py | 84 ++++++ 7 files changed, 577 insertions(+) create mode 100644 .github/workflows/build_and_publish.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 funcprofiler/__init__.py create mode 100644 setup.py create mode 100644 test.py diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml new file mode 100644 index 0000000..4a9ec6b --- /dev/null +++ b/.github/workflows/build_and_publish.yml @@ -0,0 +1,34 @@ +name: Publish Python 🐍 distributions 📦 to PyPI + +on: + push: + tags: + - '*' + +jobs: + build-n-publish: + name: Build and publish Python 🐍 distributions 📦 to PyPI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Set up Python 3.12 + uses: actions/setup-python@v3 + with: + python-version: '3.12' + - name: Install pypa/setuptools + run: >- + python -m + pip install setuptools wheel + - name: Extract tag name + id: tag + run: echo ::set-output name=TAG_NAME::$(echo $GITHUB_REF | cut -d / -f 3) + - name: Update version in setup.py + run: >- + sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" setup.py + - name: Build a binary wheel + run: >- + python setup.py sdist bdist_wheel + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..27ee28f --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__ +*.pyc +dist/ +build/ +venv/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5d7acdb --- /dev/null +++ b/LICENSE @@ -0,0 +1,7 @@ +MIT License (Modified) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to make derivative works based on the Software, provided that any substantial changes to the Software are clearly distinguished from the original work and are distributed under a different name. + +The original copyright notice and disclaimer must be retained in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS," WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT, OR OTHERWISE, ARISING FROM, OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..ad24ecc --- /dev/null +++ b/README.md @@ -0,0 +1,85 @@ +# FuncProfiler +![Python Version](https://img.shields.io/badge/python-3.12-blue.svg) +[![Code Size](https://img.shields.io/github/languages/code-size/infinitode/funcprofiler)](https://github.com/infinitode/funcprofiler) +![Downloads](https://pepy.tech/badge/funcprofiler) +![License Compliance](https://img.shields.io/badge/license-compliance-brightgreen.svg) +![PyPI Version](https://img.shields.io/pypi/v/funcprofiler) + +An open-source Python library for identifying bottlenecks in code. It includes function profiling, data exports, logging, and line-by-line profiling for more granular control. + +## Installation + +You can install FuncProfiler using pip: + +```bash +pip install funcprofiler +``` + +## Supported Python Versions + +FuncProfiler supports the following Python versions: + +- Python 3.6 +- Python 3.7 +- Python 3.8 +- Python 3.9 +- Python 3.10 +- Python 3.11 and later (preferred) + +Please ensure that you have one of these Python versions installed. FuncProfiler may not function as expected on earlier versions. + +## Features + +- **Function Profiling**: Monitor a function's memory usage and execution time to identify performance issues. +- **Line-by-Line Profiling**: Return execution time and memory usage for each line of any given function. +- **Shared Logging**: Log outputs of functions triggered by the line-by-line and function profilers, storing results in a `.txt` file. +- **File Exports**: Export profiling data from functions in `csv`, `json`, or `html` formats. +> [!NOTE] +> View more export types in the [official documentation](https://infinitode-docs.gitbook.io/documentation/package-documentation/funcprofiler-package-documentation). + +## Usage + +### Function Profiling + +```python +from funcprofiler import function_profile + +# Exporting as `html` with logging enabled +@function_profile(export_format="html", shared_log=True) +def some_function(): + return "Hello World." + +# Call the function +message = some_function() +``` + +### Line-by-Line Profiling + +```python +from funcprofiler import line_by_line_profile + +# Logging enabled without exports +@line_by_line_profile(shared_log=True) +def some_complicated_function(n): + total = 0 + for i in range(n): + for j in range(i): + total += (i * j) ** 0.5 # Square root calculation + return total + +# Call the function +total = some_complicated_function(1000) +``` + +> [!NOTE] +> FuncProfiler can be added to any function using the callable format: `@funcprofiler_function_name(expected_arguments)`. + +## Contributing + +Contributions are welcome! If you encounter issues, have suggestions, or wish to contribute to FuncProfiler, please open an issue or submit a pull request on [GitHub](https://github.com/infinitode/funcprofiler). + +## License + +FuncProfiler is released under the terms of the **MIT License (Modified)**. Please see the [LICENSE](https://github.com/infinitode/funcprofiler/blob/main/LICENSE) file for the full text. + +**Modified License Clause**: The modified license clause allows users to create derivative works based on the FuncProfiler software. However, it requires that any substantial changes to the software be clearly distinguished from the original work and distributed under a different name. \ No newline at end of file diff --git a/funcprofiler/__init__.py b/funcprofiler/__init__.py new file mode 100644 index 0000000..a58339a --- /dev/null +++ b/funcprofiler/__init__.py @@ -0,0 +1,336 @@ +import time +import tracemalloc +import functools +import inspect +import os +import sys +import csv +import json +from typing import Callable, List, Dict, Optional + +__all__ = ['function_profile', 'line_by_line_profile', 'export_function_profile_data', 'export_profiling_data'] + +def function_profile(export_format: Optional[str] = None, filename: Optional[str] = None, shared_log: bool = False) -> Callable: + """Decorator factory to profile the execution time and memory usage of a function. + + Args: + export_format (Optional[str]): The format to export the profiling data ('txt', 'json', 'csv', 'html'). + filename (Optional[str]): The name of the output file (without extension). + shared_log (Optional[bool]): If True, log to a shared file for all profiled functions. + + Returns: + Callable: The profiling wrapper or decorator function. + """ + log_filename = f"func_profiler_logs_{time.strftime('%Y%m%d')}_{time.strftime('%H%M%S')}.txt" + + def decorator(func: Callable) -> Callable: + @functools.wraps(func) + def wrapper(*args, **kwargs): + # Start time and memory tracking + tracemalloc.start() + start_time = time.time() + + # Prepare shared logging if enabled + log_file = None + if shared_log: + log_file = open(log_filename, 'a') + log_file.write(f"Profiling log for {func.__name__}\n") + log_file.write(f"Date: {time.strftime('%Y-%m-%d')}\n") + log_file.write(f"Time: {time.strftime('%H:%M:%S')}\n\n") + + # Execute the function + result = func(*args, **kwargs) + + # End time and memory tracking + end_time = time.time() + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + # Prepare data for exporting + profiling_data = { + "execution_times": end_time - start_time, + "memory_usage": current / 10**6, # Convert to MB + } + + if export_format: + file = filename or f"{func.__name__}_funcprofile_report" + export_function_profile_data(profiling_data, func, export_format, file) + + # Display the profiling results + print(f"[FUNCPROFILER] Function '{func.__name__}' executed in {end_time - start_time:.12f}s") + print(f"[FUNCPROFILER] Current memory usage: {current / 10**6:.12f}MB; Peak: {peak / 10**6:.12f}MB") + + # Log the profiling data + if shared_log and log_file: + log_file.write(f"Function {func.__name__} called with args: {args}, kwargs: {kwargs}\n") + log_file.write(f"Execution Time: {end_time - start_time:.12f}s, Memory usage: {current / 10**6:.6f}MB; Peak: {peak / 10**6:.6f}MB\n") + log_file.write("-" * 40 + "\n") # Separator between calls + + if shared_log and log_file: + log_file.close() # Close the log file after writing if shared_log is True + + return result + + return wrapper + + return decorator + +def export_function_profile_data(profiling_data: dict, func: Callable, export_format: str, filename: str) -> None: + """Export profiling data for the function profile to the specified format. + + Args: + profiling_data (dict): The profiling data containing execution time and memory usage. + func (Callable): The function that was profiled. + export_format (str): The format for export ('txt', 'json', 'csv', 'html'). + filename (str): The output filename without extension. + """ + execution_time = profiling_data["execution_times"] + memory_usage = profiling_data["memory_usage"] + + if export_format == "txt": + with open(f"{filename}.txt", 'w') as f: + f.write(f"Function: {func.__name__}\n") + f.write(f"Execution Time: {execution_time:.12f}s\n") + f.write(f"Memory Usage: {memory_usage:.6f}MB\n") + + elif export_format == "json": + with open(f"{filename}.json", 'w') as f: + json.dump({ + "function": func.__name__, + "execution_time": execution_time, + "memory_usage": memory_usage + }, f, indent=4) + + elif export_format == "csv": + with open(f"{filename}.csv", 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(["Function", "Execution Time (s)", "Memory Usage (MB)"]) + writer.writerow([func.__name__, execution_time, memory_usage]) + + elif export_format == "html": + with open(f"{filename}.html", 'w') as f: + f.write("Function Profiling Report") + f.write(f"

Function: {func.__name__}

") + f.write(f"

Execution Time: {execution_time:.12f}s

") + f.write(f"

Memory Usage: {memory_usage:.6f}MB

") + f.write("") + + else: + raise ValueError("Unsupported export format. Use 'txt', 'json', 'csv', or 'html'.") + +def line_by_line_profile( + export_format: Optional[str] = None, + filename: Optional[str] = None, + shared_log: bool = False +) -> Callable: + """Decorator for line-by-line profiling of a function with optional data export and shared logging. + + Args: + export_format (Optional[str]): The format to export the profiling data ('json', 'csv', 'html'). + filename (Optional[str]): The name of the output file (without extension). + shared_log (Optional[bool]): If True, log to a shared file for all profiled functions. + + Returns: + Callable: The profiling wrapper or decorator function. + """ + log_filename = f"lbl_profiler_logs_{time.strftime('%Y%m%d')}_{time.strftime('%H%M%S')}.txt" + + def decorator(func: Callable) -> Callable: + @functools.wraps(func) + def wrapper(*args, **kwargs): + line_execution_times: Dict[int, float] = {} + line_memory_usage: Dict[int, float] = {} + current_line_start_time: Optional[float] = None + timer = time.perf_counter + tracemalloc.start() + + # Prepare shared logging if enabled + log_file = None + if shared_log: + log_file = open(log_filename, 'a') + log_file.write(f"Profiling log for {func.__name__}\n") + log_file.write(f"Date: {time.strftime('%Y-%m-%d')}\n") + log_file.write(f"Time: {time.strftime('%H:%M:%S')}\n\n") + + def trace_lines(frame, event, arg): + nonlocal current_line_start_time + if frame.f_code.co_name == func.__name__: + lineno = frame.f_lineno + if event == 'line': + current_memory = tracemalloc.get_traced_memory()[1] / 10**6 # Convert to MB + if current_line_start_time is not None: + elapsed_time = timer() - current_line_start_time + if lineno in line_execution_times: + line_execution_times[lineno] += elapsed_time + else: + line_execution_times[lineno] = elapsed_time + line_memory_usage[lineno] = current_memory + + # Log the profiling data + if shared_log and log_file: + log_file.write(f"Line {lineno}: Execution Time: {elapsed_time:.12f}s, Memory Usage: {current_memory:.12f}MB\n") + + current_line_start_time = timer() + return trace_lines + + sys.settrace(trace_lines) + try: + result = func(*args, **kwargs) + finally: + sys.settrace(None) + tracemalloc.stop() + + # Print profiling data + print(f"\nLine-by-Line Profiling for '{func.__name__}':") + source_lines, starting_line = inspect.getsourcelines(func) + for line_no in sorted(line_execution_times.keys()): + actual_line = line_no - starting_line + 1 + source_line = source_lines[actual_line - 1].strip() + exec_time = line_execution_times[line_no] + mem_usage = line_memory_usage.get(line_no, 0) + print(f"Line {line_no} ({source_line}): " + f"Execution Time: {exec_time:.12f}s, " + f"Memory Usage: {mem_usage:.12f}MB") + + # Collect the profiling data for the report + profiling_data = { + "line_execution_times": line_execution_times, + "line_memory_usage": line_memory_usage + } + + if export_format: + file = filename or f"{func.__name__}_lblprofile_report" + export_profiling_data(profiling_data, func, export_format, file) + + # Close the log file if shared logging was enabled + if shared_log and log_file: + log_file.write("\n") # Add a new line for separation + log_file.write("-" * 40 + "\n") # Separator between calls + log_file.write("\n") # Add a new line for separation + log_file.close() + + return result + + return wrapper + + # If no arguments are provided, it means func is passed directly + if callable(export_format): + actual_func = export_format + export_format = None + return decorator(actual_func) + + return decorator + +def export_profiling_data( + profiling_data: Dict[str, Dict[int, float]], + func: Callable, + export_format: str, + filename: str +) -> None: + """Export the profiling data to the specified format (JSON, CSV, HTML). + + Args: + profiling_data (Dict[str, Dict[int, float]]): Profiling data to be exported. + func (Callable): The function that was profiled. + export_format (str): The format for export ('json', 'csv', 'html'). + filename (str): The output filename without extension. + """ + line_execution_times = profiling_data["line_execution_times"] + line_memory_usage = profiling_data["line_memory_usage"] + + # Get the source code of the function + source_lines, starting_line = inspect.getsourcelines(func) + + # Prepare data for export + export_data: List[Dict[str, str]] = [] + for line_no in sorted(line_execution_times.keys()): + actual_line = line_no - starting_line + 1 + source_line = source_lines[actual_line - 1].strip() + exec_time = line_execution_times[line_no] + mem_usage = line_memory_usage.get(line_no, 0) + + # Conditional wrapping based on the export format + wrapped_source_code = f'"{source_line}"' if export_format == 'csv' else source_line + + export_data.append({ + 'Function Name': func.__name__, + 'Line Number': str(line_no), + 'Source Code': wrapped_source_code, # Use wrapped source code + 'Execution Time (s)': f"{exec_time:.12f}", + 'Memory Usage (MB)': f"{mem_usage:.12f}" + }) + + # Handle different export formats + if export_format == 'json': + output_path = f"{filename}.json" + # Check if the file exists to append or create + if os.path.exists(output_path): + with open(output_path, 'r') as json_file: + existing_data = json.load(json_file) + existing_data.extend(export_data) + else: + existing_data = export_data + + with open(output_path, 'w') as json_file: + json.dump(existing_data, json_file, indent=4) + print(f"[PROFILER] JSON report generated at: {output_path}") + + elif export_format == 'csv': + output_path = f"{filename}.csv" + # Check if the file exists to determine write mode + file_mode = 'a' if os.path.exists(output_path) else 'w' + + with open(output_path, mode=file_mode, newline='') as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=['Function Name', 'Line Number', 'Source Code', 'Execution Time (s)', 'Memory Usage (MB)']) + if file_mode == 'w': + writer.writeheader() # Write header only for new file + for data in export_data: + writer.writerow(data) + print(f"[PROFILER] CSV report generated at: {output_path}") + + elif export_format == 'html': + output_path = f"{filename}.html" + file_mode = 'a' if os.path.exists(output_path) else 'w' + + if file_mode == 'w': + html_content = """ + + Line-by-Line Profiling Report + +

Line-by-Line Profiling Report

+ + + + + + + + + """ + else: + html_content = "" + + for data in export_data: + html_content += f""" + + + + + + + + """ + + if file_mode == 'w': + html_content += """ +
Function NameLine NumberSource CodeExecution Time (s)Memory Usage (MB)
{data['Function Name']}{data['Line Number']}{data['Source Code']}{data['Execution Time (s)']}{data['Memory Usage (MB)']}
+ + + """ + + with open(output_path, mode='a') as f: + f.write(html_content) + print(f"[PROFILER] HTML report generated at: {output_path}") + + else: + print(f"[PROFILER] Unsupported export format: {export_format}") \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..56c2f5b --- /dev/null +++ b/setup.py @@ -0,0 +1,26 @@ +from setuptools import setup, find_packages + +setup( + name='funcprofiler', + version='{{VERSION_PLACEHOLDER}}', + author='Infinitode Pty Ltd', + author_email='infinitode.ltd@gmail.com', + description='An open-source Python library that can be used to profile functions.', + long_description='An open-source Python library for finding bottlenecks in code. Includes function profiling, data exports, logging, and even line-by-line profiling, for more control.', + long_description_content_type='text/markdown', + url='https://github.com/infinitode/valx', + packages=find_packages(), + classifiers=[ + 'Development Status :: 5 - Production', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + ], + python_requires='>=3.6', +) diff --git a/test.py b/test.py new file mode 100644 index 0000000..f2065fc --- /dev/null +++ b/test.py @@ -0,0 +1,84 @@ +import unittest +import time +from funcprofiler import ( + function_profile, + line_by_line_profile, +) + +# Sample complex functions to be profiled +@line_by_line_profile(export_format="json", shared_log=False) +def complex_calculations(n): + """A complex calculation involving nested loops.""" + total = 0 + for i in range(n): + for j in range(i): + total += (i * j) ** 0.5 # Square root calculation + return total + +@line_by_line_profile(shared_log=True) +def conditional_logic(n): + """A function that uses conditional statements to manipulate a list.""" + result = [] + for i in range(n): + if i % 3 == 0: + result.append(i * 2) + elif i % 5 == 0: + result.append(i * 3) + else: + result.append(i) + return result + +@line_by_line_profile(shared_log=True) +def function_calls(n): + """A function that calls a helper function to compute squares.""" + def helper(x): + return x * x # Example helper function + + total = 0 + for i in range(n): + total += helper(i) + return total + +@line_by_line_profile(export_format="csv", filename="test01") +def simulated_io_operations(n): + """Simulates I/O operations by sleeping and calculating a sum.""" + total = 0 + for i in range(n): + if i % 2 == 0: + time.sleep(0.01) # Simulate a blocking I/O operation + total += i + return total + +@function_profile(export_format="html", shared_log=True) +def factorial(n): + """Computes the factorial of a number.""" + if n == 0: + return 1 + return n * factorial(n - 1) + +class TestFuncProfiler(unittest.TestCase): + + def test_complex_calculations(self): + result = complex_calculations(10) + self.assertAlmostEqual(result, 163.8608281556458, places=5) + + def test_conditional_logic(self): + result = conditional_logic(10) + expected = [0, 1, 2, 6, 4, 15, 12, 7, 8, 18] + self.assertEqual(result, expected) + + def test_function_calls(self): + result = function_calls(10) + self.assertEqual(result, sum(i * i for i in range(10))) + + def test_simulated_io_operations(self): + result = simulated_io_operations(10) + expected = sum(i for i in range(10) if i % 2 == 0) # Sum of even numbers from 0 to 9 + self.assertEqual(result, expected) + + def test_factorial(self): + result = factorial(5) + self.assertEqual(result, 120) # 5! = 120 + +if __name__ == '__main__': + unittest.main() \ No newline at end of file