Merge pull request #51 from avancinirodrigo/develop

Creating ecosystem feature

ecolyzer/dataaccess/__init__.py

@@ -1 +1 @@
-from .sqlalchemy_orm import SQLAlchemyORM, Base
+from .sqlalchemy_orm import SQLAlchemyORM, Base, NullSession

ecolyzer/dataaccess/sqlalchemy_orm.py

@@ -10,7 +10,7 @@ def  __init__(self, url):
 
 	def create_engine(self):
 		self.engine = create_engine(self.url)
-		self.session = sessionmaker(bind=self.engine)
+		self.session = sessionmaker(bind=self.engine, autoflush=False)
 
 	def create_all_tables(self):
 		Base.metadata.create_all(self.engine)
@@ -45,3 +45,14 @@ def drop_all(self):
 		self.dropdb() 
 
 Base = declarative_base()
+
+class NullSession:
+	"""NullSession"""
+	def add(self, arg):
+		pass
+
+	def commit(self):
+		pass
+
+	def expunge(self, arg):
+		pass

ecolyzer/ecosystem/__init__.py

@@ -0,0 +1,4 @@
+from .central_system import CentralSystem
+from .relationship import Relationship, RelationInfo
+from .ecosystem_analyzer import EcosystemAnalyzer
+from .ecosystem import Ecosystem

ecolyzer/ecosystem/central_system.py

@@ -0,0 +1,14 @@
+from sqlalchemy import Column, String, Integer, ForeignKey
+from sqlalchemy.orm import relationship, backref
+from ecolyzer.dataaccess import Base
+
+class CentralSystem(Base):
+	"""CentralSystem"""
+	__tablename__ = 'central_system'
+
+	id = Column(Integer, primary_key=True)
+	system_id = Column(Integer, ForeignKey('system.id'))
+	system = relationship('System', backref=backref('system', uselist=False))
+
+	def __init__(self, system):
+		self.system = system

ecolyzer/ecosystem/ecosystem.py

@@ -0,0 +1,17 @@
+from sqlalchemy import Column, String, Integer, ForeignKey
+from sqlalchemy.orm import relationship #, backref
+from ecolyzer.dataaccess import Base
+from .relationship import Relationship
+
+class Ecosystem(Base):
+	"""Ecosystem"""
+	__tablename__ = 'ecosystem'
+
+	id = Column(Integer, primary_key=True)
+	_relationships = relationship(Relationship) 
+
+	def add_relationship(self, relationship):
+	 	self._relationships.append(relationship)
+
+	def relationships(self):
+		return self._relationships

ecolyzer/ecosystem/ecosystem_analyzer.py

@@ -0,0 +1,28 @@
+from ecolyzer.system import System, Call, Operation
+from ecolyzer.dataaccess import NullSession
+from .relationship import Relationship, RelationInfo
+
+class EcosystemAnalyzer():
+	"""EcosystemAnalyzer"""
+
+	def __init__(self, ecosystem):
+		self.ecosystem = ecosystem
+
+	def make_relations(self, sys_from, sys_to, session=NullSession()):
+		from_source_files = sys_from.source_files
+		to_source_files = sys_to.source_files
+		for from_fullpath, from_src_file in from_source_files.items():
+			from_code_elements = from_src_file.code_elements()
+			for from_key, from_code_element in from_code_elements.items():
+				if isinstance(from_code_element, Call):
+					for to_fullpath, to_src_file in to_source_files.items():						
+						to_operation = Operation(from_code_element.name, to_src_file)
+						if to_src_file.code_element_exists(to_operation):	
+							to_code_element = to_src_file.code_element_by_key(to_operation.key) 
+							from_info = RelationInfo(sys_from, from_src_file, from_code_element)
+							to_info = RelationInfo(sys_to, to_src_file, to_code_element)
+							rel = Relationship(from_info, to_info)
+							self.ecosystem.add_relationship(rel)
+							session.add(rel)
+						session.expunge(to_operation)
+		session.commit()

ecolyzer/ecosystem/relationship.py

@@ -0,0 +1,48 @@
+from sqlalchemy import Column, String, Integer, ForeignKey
+from sqlalchemy.orm import relationship, backref
+from ecolyzer.dataaccess import Base
+#from .ecosystem import Ecosystem
+
+class Relationship(Base):
+	"""Relationship"""
+	__tablename__ = 'relationship'
+
+	id = Column(Integer, primary_key=True)
+
+	from_system_id = Column(Integer, ForeignKey('system.id'))
+	from_system = relationship('System', foreign_keys=[from_system_id])
+	from_source_file_id = Column(Integer, ForeignKey('source_file.id'))
+	from_source_file = relationship('SourceFile', foreign_keys=[from_source_file_id])
+	from_code_element_id = Column(Integer, ForeignKey('code_element.id'))
+	from_code_element = relationship('CodeElement', foreign_keys=[from_code_element_id])
+	from_author_id = Column(Integer, ForeignKey('author.id'))
+	from_author = relationship('Author', foreign_keys=[from_author_id]) 
+
+	to_system_id = Column(Integer, ForeignKey('system.id'))
+	to_system = relationship('System', foreign_keys=[to_system_id])
+	to_source_file_id = Column(Integer, ForeignKey('source_file.id'))
+	to_source_file = relationship('SourceFile', foreign_keys=[to_source_file_id])
+	to_code_element_id = Column(Integer, ForeignKey('code_element.id'))
+	to_code_element = relationship('CodeElement', foreign_keys=[to_code_element_id])		
+	to_author_id = Column(Integer, ForeignKey('author.id'))
+	to_author = relationship('Author', foreign_keys=[to_author_id]) 
+
+	ecosystem_id = Column(Integer, ForeignKey('ecosystem.id'))
+
+	def __init__(self, from_info, to_info):
+		self.from_system = from_info.system
+		self.from_source_file = from_info.source_file
+		self.from_code_element = from_info.code_element
+		self.from_author = from_info.author
+		self.to_system = to_info.system
+		self.to_source_file = to_info.source_file
+		self.to_author = to_info.author
+		self.to_code_element = to_info.code_element		
+
+class RelationInfo():
+	"""RelationInfo"""
+	def __init__(self, system, source_file, code_element):
+		self.system = system
+		self.source_file = source_file
+		self.code_element = code_element
+		self.author = code_element.author()				

ecolyzer/repository/__init__.py

@@ -4,3 +4,4 @@
 from .person import Person
 from .author import Author
 from .modification import ModificationInfo, Modification
+from .git import Git

ecolyzer/repository/git.py

@@ -0,0 +1,27 @@
+from git import Repo
+
+class Git:
+	"""Git"""
+	def __init__(self, path):
+		self.repo = Repo(path)
+		self.git_dir = self.repo.git_dir
+
+	def commit_hashs(self, max_count=None, branch='master'):
+		commits = list(self.repo.iter_commits(branch, max_count=max_count))
+		return (commit.hexsha for commit in commits)
+
+	def commit_hashs_reverse(self, max_count=None, branch='master'):
+		commits = list(self.repo.iter_commits(branch, reverse=True))		
+		if max_count:
+			return (commits[i].hexsha for i in range(max_count))
+		else:
+			return (commit.hexsha for commit in commits)
+
+	@staticmethod
+	def IsGitRepo(path):
+		try:
+			Repo(path).git_dir
+		except:
+			return False
+		return True		
+

ecolyzer/repository/modification.py

@@ -27,6 +27,9 @@ def __init__(self, mod_info, file, commit):
 		self.source_code = mod_info.source_code		
 		self.commit = commit
 		self.file = file
+
+	def author(self):
+		return self.commit.author
 
 class ModificationInfo:
 	def __init__(self, filename):

ecolyzer/repository/repository.py

@@ -3,7 +3,7 @@
 from sqlalchemy.orm.collections import attribute_mapped_collection
 from ecolyzer.dataaccess import Base
 from .author import Author
-from .repository_miner import RepositoryMiner
+from .git import Git
 
 class Repository(Base):
 	"""Repository"""
@@ -15,7 +15,7 @@ class Repository(Base):
 					collection_class=attribute_mapped_collection('email'))
 
 	def __init__(self, path):
-		if RepositoryMiner.IsGitRepo(path):
+		if Git.IsGitRepo(path):
 			self.path = path
 		else:
 			raise Exception('Invalid repository path \'{0}\''.format(path))

ecolyzer/repository/repository_miner.py

@@ -1,3 +1,5 @@
+import sys
+from sqlalchemy.orm.exc import NoResultFound
 from pydriller import RepositoryMining, GitRepository
 from pydriller.domain.commit import ModificationType
 from git import Repo
@@ -31,7 +33,8 @@ def tag_interval(self, from_tag, to_tag):
 		self.from_tag = from_tag
 		self.to_tag = to_tag
 
-	def extract(self, session, hash=None):
+	def extract(self, session, hash=None, max_count=sys.maxsize):
+		count = 0
 		for commit_driller in RepositoryMining(self.repo.path,
 							only_modifications_with_file_types=self.source_file_extensions,
 							single=hash,
@@ -42,7 +45,7 @@ def extract(self, session, hash=None):
 							only_no_merge=self.only_no_merge).traverse_commits():
 
 			commit_info = self._get_commit_info(commit_driller)
-			author = self._check_author(commit_info.author_name, commit_info.author_email)
+			author = self._check_author(session, commit_info.author_name, commit_info.author_email)
 			commit = Commit(commit_info, author, self.repo)
 			session.add(commit)
 			for mod_info in commit_info.modifications:
@@ -54,11 +57,22 @@ def extract(self, session, hash=None):
 						srcfile = self._check_source_file(file)
 						code_elements = self._extract_code_elements(srcfile, mod.source_code)
 						for element in code_elements:
-							element.modification = mod
-							session.add(element)
+							code_element = self._check_code_element(session, srcfile, element, mod)
 					session.add(mod)
+			count += 1
+			if count == max_count:
+				session.commit()
+				return	
 			session.commit()
 
+	def _check_code_element(self, session, source_file, element, modification):
+		if not source_file.code_element_exists(element):
+			source_file.add_code_element(element)
+			element.modification = modification
+		else:
+			session.expunge(element)
+		return source_file.code_element_by_key(element.key)	
+
 	def _check_source_file(self, file):
 		if self.system.source_file_exists(file.fullpath):
 			return self.system.get_source_file(file.fullpath)
@@ -87,17 +101,27 @@ def _add_file(self, fullpath):
 		self.system.add_file(file)
 		return file			
 
-	def _check_author(self, name, email):
+	def _check_author(self, session, name, email):
 		if self.repo.author_exists(email):
 			author = self.repo.get_author(email)
 			if name != author.name:
 				author.name = name 
 			return author
 		else:
-			person = Person(name, email)
-			author = Author(person)					
-			self.repo.add_author(author)
-			return author
+			author = None
+			try:
+				author = session.query(Author).\
+							filter(Person.id == Author.person_id).\
+							filter(Person.email == email).one()
+			except NoResultFound:
+				pass
+			if author:
+				return author
+			else:
+				person = Person(name, email)
+				author = Author(person)					
+				self.repo.add_author(author)
+				return author
 
 	def _is_source_file_ext(self, ext):
 		return ext in self.source_file_extensions
@@ -145,11 +169,7 @@ def is_source_file(self, file):
 
 	def extract_code_elements(self, source_file, modification):
 		return self._extract_code_elements(source_file, modification.source_code)
-
-	@staticmethod			
-	def IsGitRepo(path):
-		try:
-			Repo(path).git_dir
-		except:
-			return False
-		return True
+
+	@staticmethod
+	def HashHeadCommit(path):
+		return Repo(path).head.commit.hexsha	

ecolyzer/system/call.py

@@ -7,5 +7,4 @@ class Call(CodeElement):
 	__mapper_args__ = {'polymorphic_identity':'call'}
 
 	id = Column(None, ForeignKey('code_element.id'), primary_key=True)
-	# caller = Column(String)
 

ecolyzer/system/code_element.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, String, Integer, ForeignKey
+from sqlalchemy import Column, String, Integer, ForeignKey, UniqueConstraint
 from sqlalchemy.orm import relationship, backref
 from ecolyzer.dataaccess import Base
 
@@ -9,15 +9,21 @@ class CodeElement(Base):
 	id = Column(Integer, primary_key=True)
 	name = Column(String)
 	type = Column(String)
+	key = Column(String)
 	source_file_id = Column(Integer, ForeignKey('source_file.id'))
 	source_file = relationship('SourceFile', backref=backref('code_element', 
 											cascade='all,delete'))
 	modification_id = Column(Integer, ForeignKey('modification.id'))
 	modification = relationship('Modification', backref=backref('code_element',
 		 									cascade='all,delete'))
+	__table_args__ = (UniqueConstraint('id', 'key'),)
 	__mapper_args__ = {'polymorphic_on':type}	
 
-	def __init__(self, name, source_file=None, modification=None):
+	def __init__(self, name, source_file, modification=None):
 		self.name = name
 		self.source_file = source_file
-		self.modification = modification
+		self.modification = modification
+		self.key = self.type + '_' + self.name + '_'
+
+	def author(self):
+		return self.modification.author()