diff --git a/.gitignore b/.gitignore index ba0430d..2c2e915 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -__pycache__/ \ No newline at end of file +__pycache__/ +gcloud/flask_ecolyzer \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 815afc8..a18237d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ env: install: - pip install -r requirements.txt + - pip install -r flask_ecolyzer/requirements.txt branches: only: @@ -26,7 +27,7 @@ before_script: - psql -c "ALTER USER postgres WITH PASSWORD 'postgres'" -U postgres; script: - - pytest -v --cov=../ecolyzer + - pytest -v --cov=../ecolyzer --cov=../flask_ecolyzer after_success: - codecov -t e0ca1fb4-91fe-4111-9fb1-232f58b4e2e2 diff --git a/ecolyzer/dataaccess/sqlalchemy_orm.py b/ecolyzer/dataaccess/sqlalchemy_orm.py index 9e7e470..95830ce 100644 --- a/ecolyzer/dataaccess/sqlalchemy_orm.py +++ b/ecolyzer/dataaccess/sqlalchemy_orm.py @@ -1,6 +1,6 @@ from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import sessionmaker, scoped_session from sqlalchemy import engine from sqlalchemy_utils import database_exists, create_database, drop_database @@ -10,7 +10,7 @@ def __init__(self, url): def create_engine(self): self.engine = create_engine(self.url) - self.session = sessionmaker(bind=self.engine, autoflush=False) + self.session = scoped_session(sessionmaker(bind=self.engine, autoflush=False)) def create_all_tables(self): Base.metadata.create_all(self.engine) diff --git a/ecolyzer/ecosystem/__init__.py b/ecolyzer/ecosystem/__init__.py index d5783c4..8cf74b9 100644 --- a/ecolyzer/ecosystem/__init__.py +++ b/ecolyzer/ecosystem/__init__.py @@ -1,4 +1,4 @@ from .central_system import CentralSystem -from .relationship import Relationship, RelationInfo +from .relationship import Relationship, RelationInfo, FromRelationInfo from .ecosystem_analyzer import EcosystemAnalyzer from .ecosystem import Ecosystem diff --git a/ecolyzer/ecosystem/ecosystem_analyzer.py b/ecolyzer/ecosystem/ecosystem_analyzer.py index 4fe5105..94885d7 100644 --- a/ecolyzer/ecosystem/ecosystem_analyzer.py +++ b/ecolyzer/ecosystem/ecosystem_analyzer.py @@ -1,6 +1,7 @@ from ecolyzer.system import System, Call, Operation from ecolyzer.dataaccess import NullSession -from .relationship import Relationship, RelationInfo +from ecolyzer.parser import StaticAnalyzer +from .relationship import Relationship, RelationInfo, FromRelationInfo class EcosystemAnalyzer(): """EcosystemAnalyzer""" @@ -19,10 +20,16 @@ def make_relations(self, sys_from, sys_to, session=NullSession()): to_operation = Operation(from_code_element.name, to_src_file) if to_src_file.code_element_exists(to_operation): to_code_element = to_src_file.code_element_by_key(to_operation.key) - from_info = RelationInfo(sys_from, from_src_file, from_code_element) + from_code_element_count = self._total_of_calls(from_code_element) + from_info = FromRelationInfo(sys_from, from_src_file, + from_code_element, from_code_element_count) to_info = RelationInfo(sys_to, to_src_file, to_code_element) rel = Relationship(from_info, to_info) self.ecosystem.add_relationship(rel) session.add(rel) session.expunge(to_operation) session.commit() + + def _total_of_calls(self, from_code_element): + analyzer = StaticAnalyzer() + return analyzer.number_of_calls(from_code_element.source_code, from_code_element.name) diff --git a/ecolyzer/ecosystem/relationship.py b/ecolyzer/ecosystem/relationship.py index ab17017..1f29d30 100644 --- a/ecolyzer/ecosystem/relationship.py +++ b/ecolyzer/ecosystem/relationship.py @@ -16,7 +16,8 @@ class Relationship(Base): from_code_element_id = Column(Integer, ForeignKey('code_element.id')) from_code_element = relationship('CodeElement', foreign_keys=[from_code_element_id]) from_author_id = Column(Integer, ForeignKey('author.id')) - from_author = relationship('Author', foreign_keys=[from_author_id]) + from_author = relationship('Author', foreign_keys=[from_author_id]) + from_code_element_count = Column(Integer) to_system_id = Column(Integer, ForeignKey('system.id')) to_system = relationship('System', foreign_keys=[to_system_id]) @@ -33,6 +34,7 @@ def __init__(self, from_info, to_info): self.from_system = from_info.system self.from_source_file = from_info.source_file self.from_code_element = from_info.code_element + self.from_code_element_count = from_info.code_element_count self.from_author = from_info.author self.to_system = to_info.system self.to_source_file = to_info.source_file @@ -46,3 +48,12 @@ def __init__(self, system, source_file, code_element): self.source_file = source_file self.code_element = code_element self.author = code_element.author() + +class FromRelationInfo(RelationInfo): + """FromRelationInfo""" + def __init__(self, system, source_file, code_element, code_element_count): + self.system = system + self.source_file = source_file + self.code_element = code_element + self.author = code_element.author() + self.code_element_count = code_element_count \ No newline at end of file diff --git a/ecolyzer/parser/lua_metrics.py b/ecolyzer/parser/lua_metrics.py new file mode 100644 index 0000000..9d8f769 --- /dev/null +++ b/ecolyzer/parser/lua_metrics.py @@ -0,0 +1,10 @@ +import lizard + +class LuaMetrics(object): + """LuaMetrics""" + def __init__(self, filepath, source_code): + self.metrics = lizard.analyze_file.\ + analyze_source_code(filepath, source_code) + + def nloc(self): + return self.metrics.nloc \ No newline at end of file diff --git a/ecolyzer/parser/lua_parser.py b/ecolyzer/parser/lua_parser.py index 42a7d9e..f1b210d 100644 --- a/ecolyzer/parser/lua_parser.py +++ b/ecolyzer/parser/lua_parser.py @@ -15,7 +15,6 @@ def parser(self, src): else: raise e - def extract_functions(self): visitor = FunctionVisitor() visitor.visit(self.tree) diff --git a/ecolyzer/parser/static_analyzer.py b/ecolyzer/parser/static_analyzer.py index ce8de7f..3cc63db 100644 --- a/ecolyzer/parser/static_analyzer.py +++ b/ecolyzer/parser/static_analyzer.py @@ -1,5 +1,6 @@ from ecolyzer.system import Operation, Call from .lua_parser import LuaParser, SyntaxException, ChunkException +from .lua_metrics import LuaMetrics class StaticAnalyzer: def __init__(self): @@ -11,14 +12,15 @@ def lua_reverse_engineering(self, src_file, src): try: parser.parser(src) functions = (parser.extract_functions() - + parser.extract_local_functions() + parser.extract_table_functions()) self._remove_duplicated(functions) for func in functions: code_elements.append(Operation(func, src_file)) calls = parser.extract_calls() + parser.extract_global_calls() - self._remove_inner_calls(calls, functions) + local_functions = parser.extract_local_functions() + all_functions = functions + local_functions + self._remove_inner_calls(calls, all_functions) self._remove_duplicated(calls) for call in calls: code_elements.append(Call(call, src_file)) @@ -29,6 +31,9 @@ def lua_reverse_engineering(self, src_file, src): return code_elements + def lua_metrics(self, filepah, source_code): + return LuaMetrics(filepah, source_code) + def _remove_inner_calls(self, calls, functions): external_calls = [] for call in calls: @@ -45,4 +50,11 @@ def _remove_duplicated(self, calls): calls_aux[call] = call result.append(call) - calls[:] = result \ No newline at end of file + calls[:] = result + + def number_of_calls(self, source_code, code_element): + parser = LuaParser() + parser.parser(source_code) + calls = parser.extract_calls() + parser.extract_global_calls() + return calls.count(code_element) + \ No newline at end of file diff --git a/ecolyzer/repository/modification.py b/ecolyzer/repository/modification.py index c3470d6..da66efa 100644 --- a/ecolyzer/repository/modification.py +++ b/ecolyzer/repository/modification.py @@ -11,6 +11,7 @@ class Modification(Base): new_path = Column(String) added = Column(Integer) removed = Column(Integer) + nloc = Column(Integer) status = Column(String, nullable=False) source_code = Column(String) commit_id = Column(Integer, ForeignKey('commit.id')) @@ -23,6 +24,7 @@ def __init__(self, mod_info, file, commit): self.new_path = mod_info.new_path self.added = mod_info.added self.removed = mod_info.removed + self.nloc = mod_info.nloc self.status = mod_info.status self.source_code = mod_info.source_code self.commit = commit @@ -38,5 +40,6 @@ def __init__(self, filename): self.new_path = '' self.added = 0 self.removed = 0 + self.nloc = 0 self.status = '' self.source_code = None diff --git a/ecolyzer/repository/repository_miner.py b/ecolyzer/repository/repository_miner.py index e6eaee4..677a786 100644 --- a/ecolyzer/repository/repository_miner.py +++ b/ecolyzer/repository/repository_miner.py @@ -70,7 +70,7 @@ def extract_last_commits(self, session=NullSession(), rev=None): repo = Repo(self.repo.path) blobs = self._repo_file_blobs(repo) for blob in blobs: - commit = self._last_commit_from_path(blob.path, repo, rev) + commit = self._last_commit_from_path(blob.path, repo, rev) commit_info = self._get_commit_info_from_gitpython(commit) author = self._check_author(session, commit_info.author_name, commit_info.author_email) commit = self._check_commit(commit_info, author) @@ -97,11 +97,18 @@ def _get_modification_from_gitpython(self, blob): file_mod = ModificationInfo(blob.path) #file_mod.old_path = mod.old_path file_mod.new_path = blob.path - #file_mod.added = mod.added - #file_mod.removed = mod.removed #file_mod.status = mod.change_type.name - file_mod.source_code = self._get_blob_source_code(blob) - return file_mod + source_code = self._get_blob_source_code(blob) + file_mod.source_code = source_code + file_mod.added = self._count_lines_of_code(blob.path, source_code) + file_mod.removed = 0 + file_mod.nloc = file_mod.added + return file_mod + + def _count_lines_of_code(self, filepath, source_code): + analyzer = StaticAnalyzer() + metrics = analyzer.lua_metrics(filepath, source_code) + return metrics.nloc() def _last_commit_from_path(self, fullpath, repo, rev): return list(repo.iter_commits(rev=rev, paths=fullpath, max_count=1))[0] @@ -159,7 +166,7 @@ def _extract_current_files(self, blobs, session): def _get_blob_source_code(self, blob): data = blob.data_stream.read() - return data.decode('utf-8') + return data.decode('utf-8', errors='ignore') #TODO: handle instead ignore def _create_modification(self, source_file, source_code): #TODO: use in extract_current_files mod = ModificationInfo(mod.filename) diff --git a/ecolyzer/system/code_element.py b/ecolyzer/system/code_element.py index 9228db2..80dfab4 100644 --- a/ecolyzer/system/code_element.py +++ b/ecolyzer/system/code_element.py @@ -27,3 +27,7 @@ def __init__(self, name, source_file, modification=None): def author(self): return self.modification.author() + + @property + def source_code(self): #TODO: this is a hack, review + return self.modification.source_code \ No newline at end of file diff --git a/ecolyzer/system/file.py b/ecolyzer/system/file.py index d153450..c49b649 100644 --- a/ecolyzer/system/file.py +++ b/ecolyzer/system/file.py @@ -1,5 +1,5 @@ import os -from sqlalchemy import Column, String, Integer, ForeignKey +from sqlalchemy import Column, String, Integer, ForeignKey, UniqueConstraint from sqlalchemy.orm import relationship, backref from sqlalchemy.ext.hybrid import hybrid_property from ecolyzer.dataaccess import Base @@ -12,10 +12,12 @@ class File(Base): _name = Column('name', String, nullable=False) _path = Column('path', String) _ext = Column('ext', String) - _fullpath = Column('fullpath', String, nullable=False, unique=True) + _fullpath = Column('fullpath', String, nullable=False) system_id = Column(Integer, ForeignKey('system.id')) system = relationship('System', backref=backref('file')) + __table_args__ = (UniqueConstraint('id', 'fullpath'),) + def __init__(self, fullpath): self.fullpath = fullpath diff --git a/ecolyzer/system/source_file.py b/ecolyzer/system/source_file.py index fb59ccc..88e1995 100644 --- a/ecolyzer/system/source_file.py +++ b/ecolyzer/system/source_file.py @@ -1,4 +1,5 @@ -from sqlalchemy import Column, String, Integer, ForeignKey, PrimaryKeyConstraint +import pathlib +from sqlalchemy import Column, String, Integer, ForeignKey from sqlalchemy.orm import relationship, backref from sqlalchemy.orm.collections import attribute_mapped_collection from ecolyzer.dataaccess import Base @@ -43,9 +44,15 @@ def ext(self): def name(self): return self.file.name + def path(self): + return self.file.path + def fullpath(self): return self.file.fullpath def system(self, system): self.file.system = system - \ No newline at end of file + + @property + def source_code(self): #TODO: source code is in Modification + return open(str(pathlib.Path().absolute()) + '/' + self.file.fullpath).read() \ No newline at end of file diff --git a/flask_ecolyzer/.flaskenv b/flask_ecolyzer/.flaskenv new file mode 100644 index 0000000..97d8d12 --- /dev/null +++ b/flask_ecolyzer/.flaskenv @@ -0,0 +1 @@ +FLASK_APP=main.py \ No newline at end of file diff --git a/flask_ecolyzer/app/__init__.py b/flask_ecolyzer/app/__init__.py new file mode 100644 index 0000000..9f79f6b --- /dev/null +++ b/flask_ecolyzer/app/__init__.py @@ -0,0 +1,17 @@ +from flask import Flask, Blueprint +from flask_sqlalchemy import SQLAlchemy +from .config import Config + +db = SQLAlchemy() +bp = Blueprint('', __name__) + +def create_app(config=Config): + app = Flask(__name__) + app.config.from_object(config) + app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + db.init_app(app) + + from . import routes + app.register_blueprint(bp) + + return app diff --git a/flask_ecolyzer/app/config.py b/flask_ecolyzer/app/config.py new file mode 100644 index 0000000..3b1fa9f --- /dev/null +++ b/flask_ecolyzer/app/config.py @@ -0,0 +1,6 @@ +import os + +class Config(object): + SERVER_URL = os.environ.get('SERVER_URL') or 'http://127.0.0.1:5000' + SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') or \ + 'postgresql://postgres:postgres@localhost:5432/ecosystem_last_commits' \ No newline at end of file diff --git a/flask_ecolyzer/app/routes.py b/flask_ecolyzer/app/routes.py new file mode 100644 index 0000000..025b9b8 --- /dev/null +++ b/flask_ecolyzer/app/routes.py @@ -0,0 +1,133 @@ +from flask import jsonify, render_template, url_for +import json +from . import db +from . import bp as app +from ecolyzer.repository import Author, Modification +from ecolyzer.ecosystem import Relationship +from ecolyzer.system import Operation, SourceFile + +@app.route('/authors') +def authors(): + authors = db.session.query(Author).all() + return render_template('authors.html', authors=authors) + +@app.route('/', methods=['GET']) +@app.route('/relationships', methods=['GET']) +def relationships(): + relations = db.session.query(Relationship).all() + to_system = relations[0].to_system + relations_count = [] + source_pos = {} + paths = {} + source_ids = [] + for rel in relations: + source_id = rel.to_source_file_id + if source_id in source_pos: + pos = source_pos[source_id] + relations_count[pos]['count'] = relations_count[pos]['count'] + 1 + else: + source_pos[source_id] = len(relations_count) + operations = db.session.query(Operation).\ + filter_by(source_file_id=source_id).count() + file_mod = db.session.query(Modification).\ + filter_by(file_id = rel.to_source_file.file_id).one() + info = { + 'id': source_id, + 'source': rel.to_source_file.name(), + 'fullpath': rel.to_source_file.fullpath(), + 'path': rel.to_source_file.path(), + 'url': url_for('.source_relations', id=source_id), + 'system': rel.to_system.name, + 'operations': operations, + 'nloc': file_mod.nloc, + 'count': 1 + } + relations_count.append(info) + paths[rel.to_source_file.path()] = 0 + source_ids.append(source_id) + + sources_without_relation = db.session.query(SourceFile).\ + filter(SourceFile.id.notin_(source_ids)).\ + filter_by(system_id=to_system.id).all() + sources_without_relation_info = [] + for src in sources_without_relation: + operations = db.session.query(Operation).\ + filter_by(source_file_id=src.id).count() + if operations > 0: + file_mod = db.session.query(Modification).\ + filter_by(file_id = src.file_id).one() + info = { + 'id': src.id, + 'source': src.name(), + 'fullpath': src.fullpath(), + 'path': src.path(), + 'url': '', + 'system': to_system.name, + 'operations': operations, + 'nloc': file_mod.nloc, + 'count': 0 + } + sources_without_relation_info.append(info) + paths[src.path()] = 0 + + relations_count = relations_count + sources_without_relation_info + + return render_template('relations_count.html', relations=relations_count, + system=to_system.name, paths=paths) + +@app.route('/relationships/', methods=['GET']) +def source_relations(id): + relations = db.session.query(Relationship).filter_by(to_source_file_id = id).all() + source_file = relations[0].to_source_file + source_relations = [] + from_source_pos = {} + from_systems = {} + for rel in relations: + from_source_id = rel.from_source_file_id + if from_source_id in from_source_pos: + pos = from_source_pos[from_source_id] + source_relations[pos]['count'] += 1 + source_relations[pos]['ncalls'] += rel.from_code_element_count + else: # enter here just in the first time + from_source_pos[from_source_id] = len(source_relations) + from_systems[rel.from_system_id] = rel.from_system.name + from_source_file = rel.from_source_file + file_mod = db.session.query(Modification).\ + filter_by(file_id = from_source_file.file_id).one() + info = { + 'id': from_source_id, + 'from': from_source_file.name(), + 'fullpath': from_source_file.fullpath(), + 'code': rel.from_code_element.name + '()', + 'count': 1, + 'system': rel.from_system.name, + 'nloc': file_mod.nloc, + 'ncalls': rel.from_code_element_count, + 'url': url_for('.source_codes', from_id=from_source_id, to_id=id) + } + source_relations.append(info) + + return render_template('source_relations.html', relations=source_relations, + source_file=source_file.name(), from_systems=from_systems) + +@app.route('/relationships//', methods=['GET']) +def source_codes(from_id, to_id): + relations = db.session.query(Relationship)\ + .filter_by(to_source_file_id = to_id,\ + from_source_file_id = from_id).all() + from_file = relations[0].from_source_file.file + to_file = relations[0].to_source_file.file + from_system = relations[0].from_system.name + to_system = relations[0].to_system.name + from_source = db.session.query(Modification.source_code).\ + filter_by(file_id = from_file.id).one() + to_source = db.session.query(Modification.source_code).\ + filter_by(file_id = to_file.id).one() + code_elements = [] + for rel in relations: + # print(rel.to_code_element.name) + code_elements.append(rel.to_code_element.name) + return render_template('source_codes.html', from_source=from_source[0], + to_source=to_source[0], code_elements=code_elements, + from_fullpath=from_file.fullpath, to_fullpath=to_file.fullpath, + from_system=from_system, to_system=to_system) diff --git a/flask_ecolyzer/app/static/js/utils.js b/flask_ecolyzer/app/static/js/utils.js new file mode 100644 index 0000000..9b0e280 --- /dev/null +++ b/flask_ecolyzer/app/static/js/utils.js @@ -0,0 +1,19 @@ +function getWidth() { + return Math.max( + document.body.scrollWidth, + document.documentElement.scrollWidth, + document.body.offsetWidth, + document.documentElement.offsetWidth, + document.documentElement.clientWidth + ); +} + +function getHeight() { + return Math.max( + document.body.scrollHeight, + document.documentElement.scrollHeight, + document.body.offsetHeight, + document.documentElement.offsetHeight, + document.documentElement.clientHeight + ); +} \ No newline at end of file diff --git a/flask_ecolyzer/app/templates/authors.html b/flask_ecolyzer/app/templates/authors.html new file mode 100644 index 0000000..f7b3776 --- /dev/null +++ b/flask_ecolyzer/app/templates/authors.html @@ -0,0 +1,3 @@ +{% for author in authors %} +

{{ author.person.name + ': ' + author.person.email }}

+{% endfor %} \ No newline at end of file diff --git a/flask_ecolyzer/app/templates/relations_count.html b/flask_ecolyzer/app/templates/relations_count.html new file mode 100644 index 0000000..3e015dd --- /dev/null +++ b/flask_ecolyzer/app/templates/relations_count.html @@ -0,0 +1,325 @@ + + + + + + {{ system }} Ecosystem + + + + + +
+ + + diff --git a/flask_ecolyzer/app/templates/source_codes.html b/flask_ecolyzer/app/templates/source_codes.html new file mode 100644 index 0000000..094a51c --- /dev/null +++ b/flask_ecolyzer/app/templates/source_codes.html @@ -0,0 +1,190 @@ + + + + + Souce Codes + + + + + + + + +
+
+

{{ to_system }}/{{ to_fullpath }}

+ + +
+
+

{{ from_system }}/{{ from_fullpath }}

+ + +
+
+ + + diff --git a/flask_ecolyzer/app/templates/source_relations.html b/flask_ecolyzer/app/templates/source_relations.html new file mode 100644 index 0000000..be16d01 --- /dev/null +++ b/flask_ecolyzer/app/templates/source_relations.html @@ -0,0 +1,302 @@ + + + + + {{ source_file }} + + + + + +
+ + + + \ No newline at end of file diff --git a/flask_ecolyzer/main.py b/flask_ecolyzer/main.py new file mode 100644 index 0000000..0a23b5a --- /dev/null +++ b/flask_ecolyzer/main.py @@ -0,0 +1,3 @@ +from app import create_app + +app = create_app() diff --git a/flask_ecolyzer/requirements.txt b/flask_ecolyzer/requirements.txt new file mode 100644 index 0000000..b2423ec --- /dev/null +++ b/flask_ecolyzer/requirements.txt @@ -0,0 +1,7 @@ +flask +flask_sqlalchemy +python-dotenv +pytest-flask +blinker +pg8000 +gunicorn \ No newline at end of file diff --git a/gcloud/app.yaml b/gcloud/app.yaml new file mode 100644 index 0000000..2170ca8 --- /dev/null +++ b/gcloud/app.yaml @@ -0,0 +1,15 @@ +runtime: python37 + +handlers: +- url: /static + static_dir: app/static + +- url: /.* + script: auto + +env_variables: + PYTHONPATH: 'ecolyzer' + DATABASE_URL: postgres+pg8000://postgres:postgres@/terrame_ecosystem?unix_sock=/cloudsql/ecolyzer:us-central1:postgres/.s.PGSQL.5432 + +beta_settings: + cloud_sql_instances: ecolyzer:us-central1:postgres diff --git a/gcloud/gcloud_conf.sh b/gcloud/gcloud_conf.sh new file mode 100755 index 0000000..25b5135 --- /dev/null +++ b/gcloud/gcloud_conf.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +cp -r ../flask_ecolyzer . +cp app.yaml flask_ecolyzer +cp -r ../ecolyzer flask_ecolyzer + +cp ../requirements.txt flask_ecolyzer +cat ../flask_ecolyzer/requirements.txt >> flask_ecolyzer/requirements.txt diff --git a/scripts/terrame_ecosystem.py b/scripts/terrame_ecosystem.py new file mode 100644 index 0000000..54e2618 --- /dev/null +++ b/scripts/terrame_ecosystem.py @@ -0,0 +1,104 @@ +import os +from ecolyzer.ecosystem import EcosystemAnalyzer +from ecolyzer.system import System +from ecolyzer.repository import Repository, Person, Author, RepositoryMiner, GitPython +from ecolyzer.dataaccess import SQLAlchemyORM +from ecolyzer.ecosystem import Ecosystem + +db_url = os.environ.get('DATABASE_URL') or \ + 'postgresql://postgres:postgres@localhost:5432/terrame_ecosystem' +db = SQLAlchemyORM(db_url) +db.create_all(True) +session = db.create_session() + +repo1 = Repository('repo/terrame') +sys1 = System('TerraME', repo1) + +session.add(repo1) +session.add(sys1) + +miner = RepositoryMiner(repo1, sys1) +miner.add_ignore_dir_with('test') +miner.add_ignore_dir_with('example') +miner.add_ignore_dir_with('data') +miner.add_ignore_dir_with('images') +miner.add_ignore_dir_with('zerobrane') +miner.extract_last_commits(session, '2.0-RC-8') + +repo2 = Repository('repo/ca') +sys2 = System('ca', repo2) +miner = RepositoryMiner(repo2, sys2) +miner.extract_last_commits(session) + +repo3 = Repository('repo/calibration') +sys3 = System('calibration', repo3) +miner = RepositoryMiner(repo3, sys3) +miner.extract_last_commits(session) + +repo4 = Repository('repo/logo') +sys4 = System('logo', repo4) +miner = RepositoryMiner(repo4, sys4) +miner.extract_last_commits(session) + +repo5 = Repository('repo/rstats') +sys5 = System('rstats', repo5) +miner = RepositoryMiner(repo5, sys5) +miner.extract_last_commits(session) + +repo6 = Repository('repo/gpm') +sys6 = System('gpm', repo6) +miner = RepositoryMiner(repo6, sys6) +miner.extract_last_commits(session) + +repo7 = Repository('repo/publish') +sys7 = System('publish', repo7) +miner = RepositoryMiner(repo7, sys7) +miner.extract_last_commits(session) + +repo8 = Repository('repo/sci') +sys8 = System('sci', repo8) +miner = RepositoryMiner(repo8, sys8) +miner.extract_last_commits(session) + +repo9 = Repository('repo/urban') +sys9 = System('urban', repo9) +miner = RepositoryMiner(repo9, sys9) +miner.extract_last_commits(session) + +repo10 = Repository('repo/ford') +sys10 = System('ford', repo10) +miner = RepositoryMiner(repo10, sys10) +miner.extract_last_commits(session) + +repo11 = Repository('repo/sysdyn') +sys11 = System('sysdyn', repo11) +miner = RepositoryMiner(repo11, sys11) +miner.extract_last_commits(session) + +repo12 = Repository('repo/luccme') +sys12 = System('luccme', repo12) +miner = RepositoryMiner(repo12, sys12) +miner.extract_last_commits(session) + +repo13 = Repository('repo/inpeem') +sys13 = System('impeem', repo13) +miner = RepositoryMiner(repo13, sys13) +miner.extract_last_commits(session) + +ecosystem = Ecosystem() + +ecolyzer = EcosystemAnalyzer(ecosystem) +ecolyzer.make_relations(sys2, sys1, session) +ecolyzer.make_relations(sys3, sys1, session) +ecolyzer.make_relations(sys4, sys1, session) +ecolyzer.make_relations(sys5, sys1, session) +ecolyzer.make_relations(sys6, sys1, session) +ecolyzer.make_relations(sys7, sys1, session) +ecolyzer.make_relations(sys8, sys1, session) +ecolyzer.make_relations(sys9, sys1, session) +ecolyzer.make_relations(sys10, sys1, session) +ecolyzer.make_relations(sys11, sys1, session) +ecolyzer.make_relations(sys12, sys1, session) +ecolyzer.make_relations(sys13, sys1, session) + +session.close() \ No newline at end of file diff --git a/tests/flask_ecolyzer/conftest.py b/tests/flask_ecolyzer/conftest.py new file mode 100644 index 0000000..0f5e071 --- /dev/null +++ b/tests/flask_ecolyzer/conftest.py @@ -0,0 +1,46 @@ +import pytest +from ecolyzer.dataaccess import SQLAlchemyORM +from ecolyzer.repository import Repository, RepositoryMiner +from ecolyzer.system import System +from ecolyzer.ecosystem import Ecosystem, EcosystemAnalyzer + +@pytest.fixture(scope="module") +def db_connection(): + db_url = 'postgresql://postgres:postgres@localhost:5432/flask_test' + db = SQLAlchemyORM(db_url) + + if db.existsdb(): + yield db_connection + return + + db.create_all(True) + session = db.create_session() + + repo1 = Repository('repo/terrame') + sys1 = System('TerraME', repo1) + + session.add(repo1) + session.add(sys1) + + miner = RepositoryMiner(repo1, sys1) + miner.add_ignore_dir_with('test') + miner.add_ignore_dir_with('examples') + miner.add_ignore_dir_with('ide') + miner.add_ignore_dir_with('data') + miner.extract_last_commits(session) + + repo2 = Repository('repo/ca') + sys2 = System('CA', repo2) + miner = RepositoryMiner(repo2, sys2) + miner.extract_last_commits(session) + + ecosystem = Ecosystem() + + ecolyzer = EcosystemAnalyzer(ecosystem) + ecolyzer.make_relations(sys2, sys1, session) + + session.close() + + yield db_connection + + db.drop_all() \ No newline at end of file diff --git a/tests/flask_ecolyzer/flask_test.py b/tests/flask_ecolyzer/flask_test.py new file mode 100644 index 0000000..ab21b4a --- /dev/null +++ b/tests/flask_ecolyzer/flask_test.py @@ -0,0 +1,91 @@ +import pytest +import os +from flask import template_rendered +from contextlib import contextmanager +from flask_ecolyzer.app import create_app +from flask_ecolyzer.app.config import Config + +class TestConfig(Config): + TESTING = True + SQLALCHEMY_DATABASE_URI = 'postgresql://postgres:postgres@localhost:5432/flask_test' + +#https://stackoverflow.com/questions/39822265/flask-testing-how-to-retrieve-variables-that-were-passed-to-jinja +@contextmanager +def get_context_variables(app): + recorded = [] + def record(sender, template, context, **extra): + recorded.append(context) + template_rendered.connect(record, app) + try: + yield iter(recorded) + finally: + template_rendered.disconnect(record, app) + +@pytest.fixture +def app(): + app = create_app(TestConfig) + return app + +def test_relationships(app, client, db_connection): + with get_context_variables(app) as contexts: + res = client.get('/relationships') + contexts = next(contexts) + system = contexts['system'] + relations = contexts['relations'] + paths = contexts['paths'] + assert system == 'TerraME' + assert len(relations) == 53 + assert len(paths) == 9 + +def get_url_by_key(relations, name, key): + for rel in relations: + if rel[key] == name: + return rel['url'] + +def get_source_url(relations, name): + return get_url_by_key(relations, name, 'source') + +def get_from_url(relations, name): + return get_url_by_key(relations, name, 'from') + +def test_source_relations(app, client, db_connection): + source_relations_url = None + with get_context_variables(app) as contexts: + res = client.get('/relationships') + contexts = next(contexts) + relations = contexts['relations'] + from_system_id = relations + source_relations_url = get_source_url(relations, 'CellularSpace') + + with get_context_variables(app) as contexts: + res = client.get(source_relations_url) + contexts = next(contexts) + from_systems = contexts['from_systems'] + relations = contexts['relations'] + source_name = contexts['source_file'] + assert len(from_systems) == 1 + assert 'CA' in from_systems.values() + assert len(relations) == 32 + assert source_name == 'CellularSpace' + +def test_source_codes(app, client, db_connection): + source_relations_url = None + with get_context_variables(app) as contexts: + res = client.get('/relationships') + contexts = next(contexts) + relations = contexts['relations'] + from_system_id = relations + source_relations_url = get_source_url(relations, 'CellularSpace') + + source_codes_url = None + with get_context_variables(app) as contexts: + res = client.get(source_relations_url) + contexts = next(contexts) + relations = contexts['relations'] + source_codes_url = get_from_url(relations, 'Influenza') + + with get_context_variables(app) as contexts: + res = client.get(source_codes_url) + contexts = next(contexts) + code_elements = contexts['code_elements'] + assert len(code_elements) == 4 diff --git a/tests/integration/ecosystem_analyzer_test.py b/tests/integration/ecosystem_analyzer_test.py index b65d3d4..1716749 100644 --- a/tests/integration/ecosystem_analyzer_test.py +++ b/tests/integration/ecosystem_analyzer_test.py @@ -38,10 +38,10 @@ def test_make_relations(): relationships = ecosystem.relationships() - assert len(relationships) == 296 + assert len(relationships) == 292 rel1 = relationships[0] - rel2 = relationships[295] + rel2 = relationships[291] assert rel1.from_system.name == 'ca' assert rel1.from_author.name == 'Pedro Andrade' @@ -50,6 +50,7 @@ def test_make_relations(): assert rel1.to_author.name == 'rvmaretto' assert rel1.to_author.email == 'rvmaretto@gmail.com' assert rel1.from_code_element.name == rel1.to_code_element.name == 'createNeighborhood' + assert rel1.from_code_element_count == 1 assert rel2.from_system.name == 'ca' assert rel2.from_author.name == 'Pedro Andrade' @@ -57,12 +58,13 @@ def test_make_relations(): assert rel2.to_system.name == 'terrame' assert rel2.to_author.name == 'rvmaretto' assert rel2.to_author.email == 'rvmaretto@gmail.com' - assert rel2.from_code_element.name == rel2.to_code_element.name == 'type' - + assert rel2.from_code_element.name == rel2.to_code_element.name == 'type' + assert rel2.from_code_element_count == 1 + session.close() db.drop_all() -def test_make_relations_last_commits(): +def test_relations_last_commits(): db_url = 'postgresql://postgres:postgres@localhost:5432/ecolyzer_relations_last' db = SQLAlchemyORM(db_url) db.create_all(True) @@ -92,10 +94,10 @@ def test_make_relations_last_commits(): relationships = ecosystem.relationships() - assert len(relationships) == 575 + assert len(relationships) == 531 rel1 = relationships[0] - rel2 = relationships[574] + rel2 = relationships[530] assert rel1.from_system.name == 'ca' assert rel1.from_author.name == 'Pedro Andrade' @@ -104,6 +106,7 @@ def test_make_relations_last_commits(): assert rel1.to_author.name == 'wsenafranca' assert rel1.to_author.email == 'wsenafranca@gmail.com' assert rel1.from_code_element.name == rel1.to_code_element.name == 'run' + assert rel1.from_code_element_count == 1 assert rel2.from_system.name == 'ca' assert rel2.from_author.name == 'Pedro Andrade' @@ -112,6 +115,7 @@ def test_make_relations_last_commits(): assert rel2.to_author.name == 'Pedro Andrade' assert rel2.to_author.email == 'pedro.andrade@inpe.br' assert rel2.from_code_element.name == rel2.to_code_element.name == 'assertSnapshot' - + assert rel2.from_code_element_count == 1 + session.close() db.drop_all() \ No newline at end of file diff --git a/tests/integration/ecosystem_test.py b/tests/integration/ecosystem_test.py index 718ac27..797f9c8 100644 --- a/tests/integration/ecosystem_test.py +++ b/tests/integration/ecosystem_test.py @@ -1,7 +1,7 @@ from ecolyzer.repository import Repository, Author, Person from ecolyzer.system import System, File, SourceFile, Operation, Call from ecolyzer.dataaccess import SQLAlchemyORM -from ecolyzer.ecosystem import Relationship, RelationInfo, Ecosystem +from ecolyzer.ecosystem import Relationship, RelationInfo, FromRelationInfo, Ecosystem def test_add_relationship(mocker): db_url = 'postgresql://postgres:postgres@localhost:5432/eco_add_relation' @@ -43,11 +43,11 @@ def test_add_relationship(mocker): mocker.patch.object(c21, 'author', return_value=ca_author, autospec=True) to_info = RelationInfo(sys1, src1, c11) - from_info = RelationInfo(sys2, src2, f21) + from_info = FromRelationInfo(sys2, src2, f21, 10) rel1 = Relationship(from_info, to_info) to_info = RelationInfo(sys1, src1, f11) - from_info = RelationInfo(sys2, src2, c21) + from_info = FromRelationInfo(sys2, src2, c21, 20) rel2 = Relationship(from_info, to_info) eco = Ecosystem() @@ -70,6 +70,7 @@ def test_add_relationship(mocker): assert rel1db.to_code_element.name == 'call' assert rel1db.from_author.name == 'CA Dev' assert rel1db.to_author.name == 'TerraMe Dev' + assert rel1db.from_code_element_count == 10 rel2db = relsdb[1] assert rel2db.from_system.name == 'ca' @@ -80,6 +81,7 @@ def test_add_relationship(mocker): assert rel2db.to_code_element.name == 'get' assert rel2db.from_author.name == 'CA Dev' assert rel2db.to_author.name == 'TerraMe Dev' + assert rel2db.from_code_element_count == 20 session.close() db.drop_all() \ No newline at end of file diff --git a/tests/integration/relationship_test.py b/tests/integration/relationship_test.py index 92f7c7c..2b37ba1 100644 --- a/tests/integration/relationship_test.py +++ b/tests/integration/relationship_test.py @@ -1,7 +1,7 @@ from ecolyzer.repository import Repository, Author, Person from ecolyzer.system import System, File, SourceFile, Operation, Call from ecolyzer.dataaccess import SQLAlchemyORM -from ecolyzer.ecosystem import Relationship, RelationInfo +from ecolyzer.ecosystem import Relationship, RelationInfo, FromRelationInfo def test_crud(mocker): db_url = 'postgresql://postgres:postgres@localhost:5432/relation_crud' @@ -42,7 +42,7 @@ def test_crud(mocker): mocker.patch.object(f21, 'author', return_value=ca_author, autospec=True) to_info = RelationInfo(sys1, src1, c11) - from_info = RelationInfo(sys2, src2, f21) + from_info = FromRelationInfo(sys2, src2, f21, 10) rel = Relationship(from_info, to_info) session.add(rel) @@ -61,6 +61,7 @@ def test_crud(mocker): assert reldb.to_code_element.name == 'call' assert reldb.from_author.name == 'CA Dev' assert reldb.to_author.name == 'TerraMe Dev' + assert reldb.from_code_element_count == 10 session.close() db.drop_all() diff --git a/tests/integration/repository_miner_test.py b/tests/integration/repository_miner_test.py index 3d47d10..ea5ead1 100644 --- a/tests/integration/repository_miner_test.py +++ b/tests/integration/repository_miner_test.py @@ -210,12 +210,7 @@ def test_extract(): operations = { 'CellularSpace' : True, - 'coordCoupling' : True, - 'createMooreNeighborhood' : True, - 'createVonNeumannNeighborhood' : True, 'createNeighborhood' : True, - 'createMxNNeighborhood' : True, - 'spatialCoupling' : True, 'add' : True, 'getCell' : True, 'get' : True, @@ -238,9 +233,14 @@ def test_extract(): '__len' : True } + operationsdb_dict = {} for op in operationsdb: + operationsdb_dict[op.name] = True assert operations[op.name] + for k in operations.keys(): + assert operationsdb_dict[k] + calls = { 'addNeighborhood' : True, 'addCell' : True, @@ -301,10 +301,14 @@ def test_extract(): } callsdb = session.query(Call).filter_by(source_file_id = srcfiledb.id).all() - + callsdb_dict = {} for call in callsdb: + callsdb_dict[call.name] = True assert calls[call.name] + for k in calls.keys(): + assert callsdb_dict[k] + session.close() db.drop_all() @@ -336,7 +340,7 @@ def test_get_commit_source_file(): srcfiledb = session.query(SourceFile).filter_by(file_id = afile.id).first() assert srcfiledb.ext() == 'lua' assert srcfiledb.name() == 'CellularSpace' - assert srcfiledb.code_elements_len() == 83 + assert srcfiledb.code_elements_len() == 78 functions = session.query(Operation).filter_by(source_file_id = srcfiledb.id).all() assert srcfiledb.code_element_exists(functions[0]) @@ -542,5 +546,10 @@ def test_extract_last_commits(): assert code_elements[srcfile.code_element_by_key(k).name] assert srcfile.code_elements_len() == 40 + file_mod = session.query(Modification).\ + filter_by(file_id = srcfile.file_id).one() + + assert file_mod.nloc == file_mod.added == 165 + session.close() db.drop_all() \ No newline at end of file diff --git a/tests/integration/source_file_test.py b/tests/integration/source_file_test.py index eb28601..83d1807 100644 --- a/tests/integration/source_file_test.py +++ b/tests/integration/source_file_test.py @@ -1,3 +1,4 @@ +import os import pytest from ecolyzer.repository import Repository from ecolyzer.system import File, SourceFile, Operation, Call @@ -124,4 +125,10 @@ def test_add_same_code_element(): assert (('Code element \'get\' of type \'Operation\' is already present') in str(e.value)) session.close() - db.drop_all() \ No newline at end of file + db.drop_all() + +def test_source_code(): + filepath = 'repo/terrame/packages/base/lua/Cell.lua' + file = File(filepath) + src_file = SourceFile(file) + assert src_file.source_code != None diff --git a/tests/integration/static_analyzer_test.py b/tests/integration/static_analyzer_test.py index d741528..91d61b3 100644 --- a/tests/integration/static_analyzer_test.py +++ b/tests/integration/static_analyzer_test.py @@ -6,12 +6,7 @@ def test_lua_reverse_engineering(): operations = { 'CellularSpace' : True, - 'coordCoupling' : True, - 'createMooreNeighborhood' : True, - 'createVonNeumannNeighborhood' : True, 'createNeighborhood' : True, - 'createMxNNeighborhood' : True, - 'spatialCoupling' : True, 'add' : True, 'getCell' : True, 'get' : True, @@ -100,6 +95,16 @@ def test_lua_reverse_engineering(): analyzer = StaticAnalyzer() code_elements = analyzer.lua_reverse_engineering(src_file, src) + code_elements_dict = {} + for element in code_elements: + code_elements_dict[element.name] = True + + for k in operations.keys(): + assert code_elements_dict[k] + + for k in calls.keys(): + assert code_elements_dict[k] + assert len(code_elements) == len(operations) + len(calls) for element in code_elements: @@ -109,4 +114,13 @@ def test_lua_reverse_engineering(): else: assert operations[element.name] assert element.name not in calls - \ No newline at end of file + +def test_number_of_calls(): + luafile = os.path.join(os.path.dirname(__file__), 'data', 'CellularSpace1.lua') + file = File(luafile) + src_file = SourceFile(file) + src = open(luafile).read() + analyzer = StaticAnalyzer() + assert analyzer.number_of_calls(src, 'forEachCell') == 16 + assert analyzer.number_of_calls(src, 'addNeighborhood') == 8 + assert analyzer.number_of_calls(src, 'Cell') == 2 \ No newline at end of file diff --git a/tests/unit/ecosystem_analyzer_test.py b/tests/unit/ecosystem_analyzer_test.py index a33b366..093692a 100644 --- a/tests/unit/ecosystem_analyzer_test.py +++ b/tests/unit/ecosystem_analyzer_test.py @@ -40,6 +40,8 @@ def test_make_relations(mocker): mocker.patch.object(c1, 'author', return_value=ca_author, autospec=True) mocker.patch.object(c2, 'author', return_value=ca_author, autospec=True) + mocker.patch.object(EcosystemAnalyzer, '_total_of_calls', return_value=10) + ecolyzer = EcosystemAnalyzer(ecosystem) ecolyzer.make_relations(sys2, sys1) diff --git a/tests/unit/ecosystem_test.py b/tests/unit/ecosystem_test.py index 3161251..5bf0699 100644 --- a/tests/unit/ecosystem_test.py +++ b/tests/unit/ecosystem_test.py @@ -1,6 +1,6 @@ from ecolyzer.system import System, File, SourceFile, Operation, Call from ecolyzer.repository import Repository, Person, Author, GitPython -from ecolyzer.ecosystem import Ecosystem, RelationInfo, Relationship +from ecolyzer.ecosystem import Ecosystem, RelationInfo, FromRelationInfo, Relationship def test_add_same_relation(mocker): mocker.patch.object(GitPython, 'IsGitRepo', return_value=True) @@ -40,7 +40,7 @@ def test_add_same_relation(mocker): mocker.patch.object(c2, 'author', return_value=ca_author, autospec=True) to_info = RelationInfo(sys1, src1, op1) - from_info = RelationInfo(sys2, src3, c1) + from_info = FromRelationInfo(sys2, src3, c1, 10) rel1 = Relationship(from_info, to_info) rel2 = Relationship(from_info, to_info)