Merge pull request #59 from avancinirodrigo/develop

Creating a system MVP

.gitignore

@@ -1 +1,2 @@
-__pycache__/
+__pycache__/
+gcloud/flask_ecolyzer

.travis.yml

@@ -13,6 +13,7 @@ env:
 
 install:
   - pip install -r requirements.txt
+  - pip install -r flask_ecolyzer/requirements.txt
 
 branches:
   only:
@@ -26,7 +27,7 @@ before_script:
   - psql -c "ALTER USER postgres WITH PASSWORD 'postgres'" -U postgres;
 
 script:
-  - pytest -v --cov=../ecolyzer
+  - pytest -v --cov=../ecolyzer --cov=../flask_ecolyzer
 
 after_success:
   - codecov -t e0ca1fb4-91fe-4111-9fb1-232f58b4e2e2

ecolyzer/dataaccess/sqlalchemy_orm.py

@@ -1,6 +1,6 @@
 from sqlalchemy import create_engine
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import sessionmaker, scoped_session
 from sqlalchemy import engine
 from sqlalchemy_utils import database_exists, create_database, drop_database
 
@@ -10,7 +10,7 @@ def  __init__(self, url):
 
 	def create_engine(self):
 		self.engine = create_engine(self.url)
-		self.session = sessionmaker(bind=self.engine, autoflush=False)
+		self.session = scoped_session(sessionmaker(bind=self.engine, autoflush=False))
 
 	def create_all_tables(self):
 		Base.metadata.create_all(self.engine)

ecolyzer/ecosystem/__init__.py

@@ -1,4 +1,4 @@
 from .central_system import CentralSystem
-from .relationship import Relationship, RelationInfo
+from .relationship import Relationship, RelationInfo, FromRelationInfo
 from .ecosystem_analyzer import EcosystemAnalyzer
 from .ecosystem import Ecosystem

ecolyzer/ecosystem/ecosystem_analyzer.py

@@ -1,6 +1,7 @@
 from ecolyzer.system import System, Call, Operation
 from ecolyzer.dataaccess import NullSession
-from .relationship import Relationship, RelationInfo
+from ecolyzer.parser import StaticAnalyzer
+from .relationship import Relationship, RelationInfo, FromRelationInfo
 
 class EcosystemAnalyzer():
 	"""EcosystemAnalyzer"""
@@ -19,10 +20,16 @@ def make_relations(self, sys_from, sys_to, session=NullSession()):
 						to_operation = Operation(from_code_element.name, to_src_file)
 						if to_src_file.code_element_exists(to_operation):	
 							to_code_element = to_src_file.code_element_by_key(to_operation.key) 
-							from_info = RelationInfo(sys_from, from_src_file, from_code_element)
+							from_code_element_count = self._total_of_calls(from_code_element)
+							from_info = FromRelationInfo(sys_from, from_src_file, 
+											from_code_element, from_code_element_count)
 							to_info = RelationInfo(sys_to, to_src_file, to_code_element)
 							rel = Relationship(from_info, to_info)
 							self.ecosystem.add_relationship(rel)
 							session.add(rel)
 						session.expunge(to_operation)
 		session.commit()
+
+	def _total_of_calls(self, from_code_element):
+		analyzer = StaticAnalyzer()
+		return analyzer.number_of_calls(from_code_element.source_code, from_code_element.name)

ecolyzer/ecosystem/relationship.py

@@ -16,7 +16,8 @@ class Relationship(Base):
 	from_code_element_id = Column(Integer, ForeignKey('code_element.id'))
 	from_code_element = relationship('CodeElement', foreign_keys=[from_code_element_id])
 	from_author_id = Column(Integer, ForeignKey('author.id'))
-	from_author = relationship('Author', foreign_keys=[from_author_id]) 
+	from_author = relationship('Author', foreign_keys=[from_author_id])
+	from_code_element_count = Column(Integer) 
 
 	to_system_id = Column(Integer, ForeignKey('system.id'))
 	to_system = relationship('System', foreign_keys=[to_system_id])
@@ -33,6 +34,7 @@ def __init__(self, from_info, to_info):
 		self.from_system = from_info.system
 		self.from_source_file = from_info.source_file
 		self.from_code_element = from_info.code_element
+		self.from_code_element_count = from_info.code_element_count
 		self.from_author = from_info.author
 		self.to_system = to_info.system
 		self.to_source_file = to_info.source_file
@@ -46,3 +48,12 @@ def __init__(self, system, source_file, code_element):
 		self.source_file = source_file
 		self.code_element = code_element
 		self.author = code_element.author()				
+
+class FromRelationInfo(RelationInfo):
+	"""FromRelationInfo"""
+	def __init__(self, system, source_file, code_element, code_element_count):
+		self.system = system
+		self.source_file = source_file
+		self.code_element = code_element
+		self.author = code_element.author()
+		self.code_element_count = code_element_count

ecolyzer/parser/lua_metrics.py

@@ -0,0 +1,10 @@
+import lizard
+
+class LuaMetrics(object):
+	"""LuaMetrics"""
+	def __init__(self, filepath, source_code):
+		self.metrics = lizard.analyze_file.\
+						analyze_source_code(filepath, source_code)
+
+	def nloc(self):
+		return self.metrics.nloc

ecolyzer/parser/lua_parser.py

@@ -15,7 +15,6 @@ def parser(self, src):
 			else:
 				raise e
 
-
 	def extract_functions(self):
 		visitor = FunctionVisitor()
 		visitor.visit(self.tree)

ecolyzer/parser/static_analyzer.py

@@ -1,5 +1,6 @@
 from ecolyzer.system import Operation, Call
 from .lua_parser import LuaParser, SyntaxException, ChunkException
+from .lua_metrics import LuaMetrics
 
 class StaticAnalyzer:
 	def __init__(self):
@@ -11,14 +12,15 @@ def lua_reverse_engineering(self, src_file, src):
 		try:
 			parser.parser(src)
 			functions = (parser.extract_functions() 
-						+ parser.extract_local_functions()
 						+ parser.extract_table_functions())
 			self._remove_duplicated(functions)
 			for func in functions:
 				code_elements.append(Operation(func, src_file))
 
 			calls = parser.extract_calls() + parser.extract_global_calls()
-			self._remove_inner_calls(calls, functions)
+			local_functions = parser.extract_local_functions()
+			all_functions = functions + local_functions
+			self._remove_inner_calls(calls, all_functions)
 			self._remove_duplicated(calls)
 			for call in calls:
 				code_elements.append(Call(call, src_file))
@@ -29,6 +31,9 @@ def lua_reverse_engineering(self, src_file, src):
 
 		return code_elements
 
+	def lua_metrics(self, filepah, source_code):
+		return LuaMetrics(filepah, source_code)
+
 	def _remove_inner_calls(self, calls, functions):
 		external_calls = []
 		for call in calls:
@@ -45,4 +50,11 @@ def _remove_duplicated(self, calls):
 				calls_aux[call] = call
 				result.append(call)
 
-		calls[:] = result
+		calls[:] = result
+
+	def number_of_calls(self, source_code, code_element):
+		parser = LuaParser()
+		parser.parser(source_code)
+		calls = parser.extract_calls() + parser.extract_global_calls()
+		return calls.count(code_element)
+

ecolyzer/repository/modification.py

@@ -11,6 +11,7 @@ class Modification(Base):
 	new_path = Column(String)
 	added = Column(Integer)
 	removed = Column(Integer)
+	nloc = Column(Integer)
 	status = Column(String, nullable=False)
 	source_code = Column(String)
 	commit_id = Column(Integer, ForeignKey('commit.id'))
@@ -23,6 +24,7 @@ def __init__(self, mod_info, file, commit):
 		self.new_path = mod_info.new_path
 		self.added = mod_info.added
 		self.removed = mod_info.removed
+		self.nloc = mod_info.nloc
 		self.status = mod_info.status
 		self.source_code = mod_info.source_code		
 		self.commit = commit
@@ -38,5 +40,6 @@ def __init__(self, filename):
 		self.new_path = ''
 		self.added = 0
 		self.removed = 0
+		self.nloc = 0
 		self.status = ''
 		self.source_code = None

ecolyzer/repository/repository_miner.py

@@ -70,7 +70,7 @@ def extract_last_commits(self, session=NullSession(), rev=None):
 		repo = Repo(self.repo.path)
 		blobs = self._repo_file_blobs(repo)
 		for blob in blobs:
-			commit = self._last_commit_from_path(blob.path, repo, rev)		
+			commit = self._last_commit_from_path(blob.path, repo, rev)	
 			commit_info = self._get_commit_info_from_gitpython(commit)
 			author = self._check_author(session, commit_info.author_name, commit_info.author_email)
 			commit = self._check_commit(commit_info, author)
@@ -97,11 +97,18 @@ def _get_modification_from_gitpython(self, blob):
 		file_mod = ModificationInfo(blob.path)
 		#file_mod.old_path = mod.old_path
 		file_mod.new_path = blob.path
-		#file_mod.added = mod.added
-		#file_mod.removed = mod.removed
 		#file_mod.status = mod.change_type.name 
-		file_mod.source_code = self._get_blob_source_code(blob)
-		return file_mod			
+		source_code = self._get_blob_source_code(blob)
+		file_mod.source_code = source_code
+		file_mod.added = self._count_lines_of_code(blob.path, source_code)
+		file_mod.removed = 0
+		file_mod.nloc = file_mod.added
+		return file_mod
+
+	def _count_lines_of_code(self, filepath, source_code):
+		analyzer = StaticAnalyzer()
+		metrics = analyzer.lua_metrics(filepath, source_code)
+		return metrics.nloc()
 
 	def _last_commit_from_path(self, fullpath, repo, rev):
 		return list(repo.iter_commits(rev=rev, paths=fullpath, max_count=1))[0]
@@ -159,7 +166,7 @@ def _extract_current_files(self, blobs, session):
 
 	def _get_blob_source_code(self, blob):
 		data = blob.data_stream.read()
-		return data.decode('utf-8')
+		return data.decode('utf-8', errors='ignore') #TODO: handle instead ignore
 
 	def _create_modification(self, source_file, source_code): #TODO: use in extract_current_files		
 		mod = ModificationInfo(mod.filename)

ecolyzer/system/code_element.py

@@ -27,3 +27,7 @@ def __init__(self, name, source_file, modification=None):
 
 	def author(self):
 		return self.modification.author()
+
+	@property
+	def source_code(self): #TODO: this is a hack, review
+		return self.modification.source_code

ecolyzer/system/file.py

@@ -1,5 +1,5 @@
 import os
-from sqlalchemy import Column, String, Integer, ForeignKey
+from sqlalchemy import Column, String, Integer, ForeignKey, UniqueConstraint
 from sqlalchemy.orm import relationship, backref
 from sqlalchemy.ext.hybrid import hybrid_property
 from ecolyzer.dataaccess import Base
@@ -12,10 +12,12 @@ class File(Base):
 	_name = Column('name', String, nullable=False)
 	_path = Column('path', String)
 	_ext = Column('ext', String)
-	_fullpath = Column('fullpath', String, nullable=False, unique=True)
+	_fullpath = Column('fullpath', String, nullable=False)
 	system_id = Column(Integer, ForeignKey('system.id'))
 	system = relationship('System', backref=backref('file'))
 
+	__table_args__ = (UniqueConstraint('id', 'fullpath'),)	
+
 	def __init__(self, fullpath):
 		self.fullpath = fullpath
 

ecolyzer/system/source_file.py

@@ -1,4 +1,5 @@
-from sqlalchemy import Column, String, Integer, ForeignKey, PrimaryKeyConstraint
+import pathlib
+from sqlalchemy import Column, String, Integer, ForeignKey
 from sqlalchemy.orm import relationship, backref
 from sqlalchemy.orm.collections import attribute_mapped_collection
 from ecolyzer.dataaccess import Base
@@ -43,9 +44,15 @@ def ext(self):
 	def name(self):
 		return self.file.name
 
+	def path(self):
+		return self.file.path
+
 	def fullpath(self):
 		return self.file.fullpath		
 
 	def system(self, system):
 		self.file.system = system
-
+
+	@property
+	def source_code(self): #TODO: source code is in Modification		
+		return open(str(pathlib.Path().absolute()) + '/' + self.file.fullpath).read()

flask_ecolyzer/.flaskenv

@@ -0,0 +1 @@
+FLASK_APP=main.py

flask_ecolyzer/app/__init__.py

@@ -0,0 +1,17 @@
+from flask import Flask, Blueprint
+from flask_sqlalchemy import SQLAlchemy
+from .config import Config
+
+db = SQLAlchemy()
+bp = Blueprint('', __name__)
+
+def create_app(config=Config):
+	app = Flask(__name__)
+	app.config.from_object(config)	
+	app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
+	db.init_app(app)
+
+	from . import routes
+	app.register_blueprint(bp)
+
+	return app

flask_ecolyzer/app/config.py

@@ -0,0 +1,6 @@
+import os
+
+class Config(object):
+	SERVER_URL = os.environ.get('SERVER_URL') or 'http://127.0.0.1:5000'
+	SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') or \
+		'postgresql://postgres:postgres@localhost:5432/ecosystem_last_commits'