From df8d1ecc6b93264794caa17a26ddd22c077bb9ec Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Sat, 11 Jun 2016 17:36:36 +0200 Subject: [PATCH 001/176] multi-schema tenancy --- gengine/__init__.py | 56 +-- gengine/alembic/env.py | 48 +-- .../alembic/versions/16b4a243d41d_nullable.py | 44 --- .../versions/2351a64b05ef_added_cascades.py | 99 ------ ...eb378_rename_properties_to_achievement_.py | 25 -- ...b1445ec_deprecate_goal_name_translation.py | 33 -- .../versions/3fd502c152c9_added_groups.py | 41 --- ...2ab7edc19e2_added_achievementcategories.py | 36 -- .../5018059c5c8f_added_goal_properties.py | 47 --- gengine/cache.py | 116 ++++++ gengine/formular.py | 67 ++++ gengine/model_base.py | 90 +++++ gengine/model_olymp.py | 46 +++ gengine/{models.py => model_tenant.py} | 195 +--------- gengine/olympadmin.py | 81 +++++ gengine/resources.py | 32 ++ gengine/scripts/initializedb.py | 332 ++++++++++-------- gengine/templates_olymp/admin/index.html | 37 ++ .../admin/layout.html | 0 gengine/templates_olymp/admin_layout.html | 12 + .../{templates => templates_olymp}/error.html | 0 .../admin/index.html | 0 gengine/templates_tenant/admin/layout.html | 81 +++++ .../admin_layout.html | 2 +- .../admin_maintenance.html | 0 gengine/templates_tenant/error.html | 20 ++ gengine/{flaskadmin.py => tenantadmin.py} | 44 +-- gengine/urlcache.py | 52 --- gengine/views.py | 47 +-- 29 files changed, 866 insertions(+), 817 deletions(-) delete mode 100644 gengine/alembic/versions/16b4a243d41d_nullable.py delete mode 100644 gengine/alembic/versions/2351a64b05ef_added_cascades.py delete mode 100644 gengine/alembic/versions/3740c3deb378_rename_properties_to_achievement_.py delete mode 100644 gengine/alembic/versions/39caab1445ec_deprecate_goal_name_translation.py delete mode 100644 gengine/alembic/versions/3fd502c152c9_added_groups.py delete mode 100644 gengine/alembic/versions/42ab7edc19e2_added_achievementcategories.py delete mode 100644 gengine/alembic/versions/5018059c5c8f_added_goal_properties.py create mode 100644 gengine/cache.py create mode 100644 gengine/formular.py create mode 100644 gengine/model_base.py create mode 100644 gengine/model_olymp.py rename gengine/{models.py => model_tenant.py} (90%) create mode 100644 gengine/olympadmin.py create mode 100644 gengine/resources.py create mode 100644 gengine/templates_olymp/admin/index.html rename gengine/{templates => templates_olymp}/admin/layout.html (100%) create mode 100644 gengine/templates_olymp/admin_layout.html rename gengine/{templates => templates_olymp}/error.html (100%) rename gengine/{templates => templates_tenant}/admin/index.html (100%) create mode 100644 gengine/templates_tenant/admin/layout.html rename gengine/{templates => templates_tenant}/admin_layout.html (94%) rename gengine/{templates => templates_tenant}/admin_maintenance.html (100%) create mode 100644 gengine/templates_tenant/error.html rename gengine/{flaskadmin.py => tenantadmin.py} (87%) delete mode 100644 gengine/urlcache.py diff --git a/gengine/__init__.py b/gengine/__init__.py index 9a96d45..2aa9758 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from pyramid import events + __version__ = '0.1.36' import datetime, os @@ -8,15 +10,22 @@ from sqlalchemy import engine_from_config -from sqlalchemy.orm import sessionmaker from pyramid.settings import asbool from gengine.wsgiutil import HTTPSProxied, init_reverse_proxy - def main(global_config, **settings): """ This function returns a Pyramid WSGI application. """ - config = Configurator(settings=settings) + engine = engine_from_config(settings, 'sqlalchemy.', connect_args={"options": "-c timezone=utc"}, ) + + from gengine.metadata import init_session, init_declarative_base, init_db + + init_session() + init_declarative_base() + init_db(engine) + + from gengine.resources import root_factory + config = Configurator(settings=settings, root_factory=root_factory) config.include('pyramid_dogpile_cache') durl = os.environ.get("DATABASE_URL") #heroku @@ -27,15 +36,8 @@ def main(global_config, **settings): if murl: settings['urlcache_url']=murl - engine = engine_from_config(settings, 'sqlalchemy.',connect_args={"options": "-c timezone=utc"},) - + config.include("pyramid_tm") - - from gengine.metadata import init_session, init_declarative_base, init_db - init_session() - init_declarative_base() - init_db(engine) - config.include('pyramid_chameleon') urlprefix = settings.get("urlprefix","") @@ -47,22 +49,28 @@ def main(global_config, **settings): urlcache_active = asbool(os.environ.get("URLCACHE_ACTIVE", settings.get("urlcache_active",True))) #routes - config.add_route('get_progress', urlprefix+'/progress/{user_id}') - config.add_route('increase_value', urlprefix+'/increase_value/{variable_name}/{user_id}') - config.add_route('increase_value_with_key', urlprefix+'/increase_value/{variable_name}/{user_id}/{key}') - config.add_route('increase_multi_values', urlprefix+'/increase_multi_values') - config.add_route('add_or_update_user', urlprefix+'/add_or_update_user/{user_id}') - config.add_route('delete_user', urlprefix+'/delete_user/{user_id}') - config.add_route('get_achievement_level', urlprefix+'/achievement/{achievement_id}/level/{level}') + config.add_route('get_progress', urlprefix+'/t/{tenant}/progress/{user_id}', traverse="/t/{tenant}") + config.add_route('increase_value', urlprefix+'/t/{tenant}/increase_value/{variable_name}/{user_id}', traverse="/t/{tenant}") + config.add_route('increase_value_with_key', urlprefix+'/t/{tenant}/increase_value/{variable_name}/{user_id}/{key}', traverse="/t/{tenant}") + config.add_route('increase_multi_values', urlprefix+'/t/{tenant}/increase_multi_values', traverse="/t/{tenant}") + config.add_route('add_or_update_user', urlprefix+'/t/{tenant}/add_or_update_user/{user_id}', traverse="/t/{tenant}") + config.add_route('delete_user', urlprefix+'/t/{tenant}/delete_user/{user_id}', traverse="/t/{tenant}") + config.add_route('get_achievement_level', urlprefix+'/t/{tenant}/achievement/{achievement_id}/level/{level}', traverse="/t/{tenant}") #config.add_route('get_achievement_reward', urlprefix+'/achievement_reward/{achievement_reward_id}') - config.add_route('admin', '/*subpath') #prefix is set in flaskadmin.py - - from gengine.flaskadmin import init_flaskadmin - init_flaskadmin(urlprefix=urlprefix, - secret=settings.get("flaskadmin_secret","fKY7kJ2xSrbPC5yieEjV")) + config.add_route('admin_tenant', '/t/{tenant}/*subpath', traverse="/t/{tenant}") #prefix is set in flaskadmin.py + + config.add_route('admin_olymp', '/olymp/*subpath') # prefix is set in flaskadmin.py + + from gengine.tenantadmin import init_admin as init_tenantadmin + init_tenantadmin(urlprefix=urlprefix, + secret=settings.get("flaskadmin_secret","fKY7kJ2xSrbPC5yieEjV")) + + from gengine.olympadmin import init_admin as init_olympadmin + init_olympadmin(urlprefix=urlprefix, + secret=settings.get("flaskadmin_secret", "fKY7kJ2xSrbPC5yieEjV")) - from .urlcache import setup_urlcache + from .cache import setup_urlcache setup_urlcache(prefix=urlprefix, url = urlcache_url, active = urlcache_active, diff --git a/gengine/alembic/env.py b/gengine/alembic/env.py index a49f0e5..861893a 100644 --- a/gengine/alembic/env.py +++ b/gengine/alembic/env.py @@ -8,11 +8,14 @@ # this is the Alembic Config object, which provides # access to the values within the .ini file in use. -config = context.config +from sqlalchemy.engine import create_engine + +config = context.config # Interpret the config file for Python logging. # This line sets up loggers basically. -fileConfig(config.config_file_name) +if config.config_file_name: + fileConfig(config.config_file_name) overrides = {} @@ -29,8 +32,6 @@ from gengine.metadata import Base target_metadata = Base.metadata -from gengine.models import * - # target_metadata = None # other values from the config, defined by the needs of env.py, @@ -38,39 +39,23 @@ # my_important_option = config.get_main_option("my_important_option") # ... etc. +def run_migrations_online(): + """Run migrations in 'online' mode. -def run_migrations_offline(): - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. + In this scenario we need to create an Engine + and associate a connection with the context. """ - url = config.get_main_option("sqlalchemy.url") - context.configure(url=url, target_metadata=target_metadata) - with context.begin_transaction(): - context.run_migrations() + engine = config.attributes["engine"] + schema = config.attributes["schema"] -def run_migrations_online(): - """Run migrations in 'online' mode. + #connectable = create_engine(url, poolclass=pool.NullPool) - In this scenario we need to create an Engine - and associate a connection with the context. + with engine.connect() as connection: - """ - connectable = engine_from_config( - config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) - - with connectable.connect() as connection: + connection.execute("SET search_path TO "+schema) context.configure( connection=connection, @@ -80,7 +65,4 @@ def run_migrations_online(): with context.begin_transaction(): context.run_migrations() -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() +run_migrations_online() diff --git a/gengine/alembic/versions/16b4a243d41d_nullable.py b/gengine/alembic/versions/16b4a243d41d_nullable.py deleted file mode 100644 index f95cc06..0000000 --- a/gengine/alembic/versions/16b4a243d41d_nullable.py +++ /dev/null @@ -1,44 +0,0 @@ -"""nullable - -Revision ID: 16b4a243d41d -Revises: 2351a64b05ef -Create Date: 2015-04-01 10:55:03.054248 - -""" - -# revision identifiers, used by Alembic. -revision = '16b4a243d41d' -down_revision = '2351a64b05ef' -branch_labels = None -depends_on = None - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.alter_column('achievements_rewards', 'achievement_id', - existing_type=sa.INTEGER(), - nullable=False) - op.alter_column('achievements_rewards', 'reward_id', - existing_type=sa.INTEGER(), - nullable=False) - op.alter_column('goals', 'achievement_id', - existing_type=sa.INTEGER(), - nullable=False) - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.alter_column('goals', 'achievement_id', - existing_type=sa.INTEGER(), - nullable=True) - op.alter_column('achievements_rewards', 'reward_id', - existing_type=sa.INTEGER(), - nullable=True) - op.alter_column('achievements_rewards', 'achievement_id', - existing_type=sa.INTEGER(), - nullable=True) - ### end Alembic commands ### diff --git a/gengine/alembic/versions/2351a64b05ef_added_cascades.py b/gengine/alembic/versions/2351a64b05ef_added_cascades.py deleted file mode 100644 index 9363e01..0000000 --- a/gengine/alembic/versions/2351a64b05ef_added_cascades.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Added Cascades - -Revision ID: 2351a64b05ef -Revises: 160e3f4be10a -Create Date: 2015-04-01 09:15:47.490122 - -""" - -# revision identifiers, used by Alembic. -revision = '2351a64b05ef' -down_revision = '3fd502c152c9' -branch_labels = None -depends_on = None - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - from sqlalchemy.engine.reflection import Inspector - insp = Inspector.from_engine(op.get_bind()) - tables = insp.get_table_names() - - for table in tables: - fks = insp.get_foreign_keys(table) - - - for fk in fks: - op.execute("ALTER TABLE "+table+" DROP CONSTRAINT "+fk["name"]) - - op.create_foreign_key(None, 'achievements', 'achievementcategories', ['achievementcategory_id'], ['id'], ondelete="SET NULL") - op.create_foreign_key(None, 'achievements_properties', 'translationvariables', ['value_translation_id'], ['id'], ondelete="RESTRICT") - op.create_foreign_key(None, 'achievements_properties', 'properties', ['property_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'achievements_properties', 'achievements', ['achievement_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'achievements_rewards', 'achievements', ['achievement_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'achievements_rewards', 'translationvariables', ['value_translation_id'], ['id'], ondelete="RESTRICT") - op.create_foreign_key(None, 'achievements_rewards', 'rewards', ['reward_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'achievements_users', 'users', ['user_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'achievements_users', 'achievements', ['achievement_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'denials', 'achievements', ['from_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'denials', 'achievements', ['to_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'goal_evaluation_cache', 'goals', ['goal_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'goal_evaluation_cache', 'users', ['user_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'goals', 'translationvariables', ['name_translation_id'], ['id'], ondelete="RESTRICT") - op.create_foreign_key(None, 'goals', 'achievements', ['achievement_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'requirements', 'achievements', ['to_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'requirements', 'achievements', ['from_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'translations', 'languages', ['language_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'translations', 'translationvariables', ['translationvariable_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'users_groups', 'users', ['user_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'users_groups', 'groups', ['group_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'users_users', 'users', ['to_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'users_users', 'users', ['from_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'values', 'users', ['user_id'], ['id'], ondelete="CASCADE") - op.create_foreign_key(None, 'values', 'variables', ['variable_id'], ['id'], ondelete="CASCADE") - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - - from sqlalchemy.engine.reflection import Inspector - insp = Inspector.from_engine(op.get_bind()) - tables = insp.get_table_names() - - for table in tables: - fks = insp.get_foreign_keys(table) - - for fk in fks: - op.execute("ALTER TABLE "+table+" DROP CONSTRAINT "+fk["name"]) - - op.create_foreign_key(None, 'achievements', 'achievementcategories', ['achievementcategory_id'], ['id']) - op.create_foreign_key(None, 'achievements_properties', 'translationvariables', ['value_translation_id'], ['id']) - op.create_foreign_key(None, 'achievements_properties', 'properties', ['property_id'], ['id']) - op.create_foreign_key(None, 'achievements_properties', 'achievements', ['achievement_id'], ['id']) - op.create_foreign_key(None, 'achievements_rewards', 'achievements', ['achievement_id'], ['id']) - op.create_foreign_key(None, 'achievements_rewards', 'translationvariables', ['value_translation_id'], ['id']) - op.create_foreign_key(None, 'achievements_rewards', 'rewards', ['reward_id'], ['id']) - op.create_foreign_key(None, 'achievements_users', 'users', ['user_id'], ['id']) - op.create_foreign_key(None, 'achievements_users', 'achievements', ['achievement_id'], ['id']) - op.create_foreign_key(None, 'denials', 'achievements', ['from_id'], ['id']) - op.create_foreign_key(None, 'denials', 'achievements', ['to_id'], ['id']) - op.create_foreign_key(None, 'goal_evaluation_cache', 'goals', ['goal_id'], ['id']) - op.create_foreign_key(None, 'goal_evaluation_cache', 'users', ['user_id'], ['id']) - op.create_foreign_key(None, 'goals', 'translationvariables', ['name_translation_id'], ['id']) - op.create_foreign_key(None, 'goals', 'achievements', ['achievement_id'], ['id']) - op.create_foreign_key(None, 'requirements', 'achievements', ['to_id'], ['id']) - op.create_foreign_key(None, 'requirements', 'achievements', ['from_id'], ['id']) - op.create_foreign_key(None, 'translations', 'languages', ['language_id'], ['id']) - op.create_foreign_key(None, 'translations', 'translationvariables', ['translationvariable_id'], ['id']) - op.create_foreign_key(None, 'users_groups', 'users', ['user_id'], ['id']) - op.create_foreign_key(None, 'users_groups', 'groups', ['group_id'], ['id']) - op.create_foreign_key(None, 'users_users', 'users', ['to_id'], ['id']) - op.create_foreign_key(None, 'users_users', 'users', ['from_id'], ['id']) - op.create_foreign_key(None, 'values', 'users', ['user_id'], ['id']) - op.create_foreign_key(None, 'values', 'variables', ['variable_id'], ['id']) - - ### end Alembic commands ### diff --git a/gengine/alembic/versions/3740c3deb378_rename_properties_to_achievement_.py b/gengine/alembic/versions/3740c3deb378_rename_properties_to_achievement_.py deleted file mode 100644 index 1aeaf5f..0000000 --- a/gengine/alembic/versions/3740c3deb378_rename_properties_to_achievement_.py +++ /dev/null @@ -1,25 +0,0 @@ -"""rename properties to achievement properties - -Revision ID: 3740c3deb378 -Revises: 5018059c5c8f -Create Date: 2015-09-23 12:23:05.418501 - -""" - -# revision identifiers, used by Alembic. -revision = '3740c3deb378' -down_revision = '5018059c5c8f' -branch_labels = None -depends_on = None - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - op.rename_table('properties','achievementproperties') - op.rename_table('achievements_properties','achievements_achievementproperties') - -def downgrade(): - op.rename_table('achievementproperties','properties') - op.rename_table('achievements_achievementproperties','achievements_properties') \ No newline at end of file diff --git a/gengine/alembic/versions/39caab1445ec_deprecate_goal_name_translation.py b/gengine/alembic/versions/39caab1445ec_deprecate_goal_name_translation.py deleted file mode 100644 index ccd8156..0000000 --- a/gengine/alembic/versions/39caab1445ec_deprecate_goal_name_translation.py +++ /dev/null @@ -1,33 +0,0 @@ -"""deprecate goal name_translation - -Revision ID: 39caab1445ec -Revises: 3740c3deb378 -Create Date: 2015-09-25 13:46:17.770482 - -""" - -# revision identifiers, used by Alembic. -revision = '39caab1445ec' -down_revision = '3740c3deb378' -branch_labels = None -depends_on = None - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.alter_column('goals', 'name_translation_id', - existing_type=sa.INTEGER(), - nullable=True) - - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.alter_column('goals', 'name_translation_id', - existing_type=sa.INTEGER(), - nullable=False) - ### end Alembic commands ### diff --git a/gengine/alembic/versions/3fd502c152c9_added_groups.py b/gengine/alembic/versions/3fd502c152c9_added_groups.py deleted file mode 100644 index 44e23f2..0000000 --- a/gengine/alembic/versions/3fd502c152c9_added_groups.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Added groups - -Revision ID: 3fd502c152c9 -Revises: 42ab7edc19e2 -Create Date: 2015-03-31 14:48:03.675985 - -""" - -# revision identifiers, used by Alembic. -revision = '3fd502c152c9' -down_revision = '42ab7edc19e2' -branch_labels = None -depends_on = None - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.create_table('groups', - sa.Column('id', sa.BigInteger(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('users_groups', - sa.Column('user_id', sa.BigInteger(), nullable=False), - sa.Column('group_id', sa.BigInteger(), nullable=False), - sa.ForeignKeyConstraint(['group_id'], ['groups.id'], ), - sa.ForeignKeyConstraint(['user_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('user_id', 'group_id') - ) - op.create_index(op.f('ix_achievements_achievementcategory_id'), 'achievements', ['achievementcategory_id'], unique=False) - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f('ix_achievements_achievementcategory_id'), table_name='achievements') - op.drop_table('users_groups') - op.drop_table('groups') - ### end Alembic commands ### diff --git a/gengine/alembic/versions/42ab7edc19e2_added_achievementcategories.py b/gengine/alembic/versions/42ab7edc19e2_added_achievementcategories.py deleted file mode 100644 index 3f42e74..0000000 --- a/gengine/alembic/versions/42ab7edc19e2_added_achievementcategories.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Added achievementcategories - -Revision ID: 42ab7edc19e2 -Revises: -Create Date: 2015-03-31 13:57:22.570668 - -""" - -# revision identifiers, used by Alembic. -revision = '42ab7edc19e2' -down_revision = None -branch_labels = None -depends_on = None - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.create_table('achievementcategories', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(length=255), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.add_column(u'achievements', sa.Column('achievementcategory_id', sa.Integer(), nullable=True)) - op.create_foreign_key(None, 'achievements', 'achievementcategories', ['achievementcategory_id'], ['id']) - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint(None, 'achievements', type_='foreignkey') - op.drop_column(u'achievements', 'achievementcategory_id') - op.drop_table('achievementcategories') - ### end Alembic commands ### diff --git a/gengine/alembic/versions/5018059c5c8f_added_goal_properties.py b/gengine/alembic/versions/5018059c5c8f_added_goal_properties.py deleted file mode 100644 index a579b76..0000000 --- a/gengine/alembic/versions/5018059c5c8f_added_goal_properties.py +++ /dev/null @@ -1,47 +0,0 @@ -"""added goal properties - -Revision ID: 5018059c5c8f -Revises: 16b4a243d41d -Create Date: 2015-09-23 11:56:01.897992 - -""" - -# revision identifiers, used by Alembic. -revision = '5018059c5c8f' -down_revision = '16b4a243d41d' -branch_labels = None -depends_on = None - -from alembic import op -import sqlalchemy as sa - - -def upgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.create_table('goalproperties', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(length=255), nullable=False), - sa.Column('is_variable', sa.Boolean(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - op.create_table('goals_goalproperties', - sa.Column('goal_id', sa.Integer(), nullable=False), - sa.Column('property_id', sa.Integer(), nullable=False), - sa.Column('value', sa.String(length=255), nullable=True), - sa.Column('value_translation_id', sa.Integer(), nullable=True), - sa.Column('from_level', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['goal_id'], ['goals.id'], ondelete='CASCADE'), - sa.ForeignKeyConstraint(['property_id'], ['goalproperties.id'], ondelete='CASCADE'), - sa.ForeignKeyConstraint(['value_translation_id'], ['translationvariables.id'], ondelete='RESTRICT'), - sa.PrimaryKeyConstraint('goal_id', 'property_id', 'from_level') - ) - op.add_column(u'goals', sa.Column('name', sa.String(length=255), nullable=False, server_default="")) - ### end Alembic commands ### - - -def downgrade(): - ### commands auto generated by Alembic - please adjust! ### - op.drop_column(u'goals', 'name') - op.drop_table('goals_goalproperties') - op.drop_table('goalproperties') - ### end Alembic commands ### diff --git a/gengine/cache.py b/gengine/cache.py new file mode 100644 index 0000000..3cc921c --- /dev/null +++ b/gengine/cache.py @@ -0,0 +1,116 @@ +import warnings +from dogpile.cache import make_region +from pyramid_dogpile_cache import get_region + + +def my_key_mangler(prefix): + def s(o): + if type(o) == dict: + return "_".join(["%s=%s" % (str(k), str(v)) for k, v in o.items()]) + if type(o) == tuple: + return "_".join([str(v) for v in o]) + if type(o) == list: + return "_".join([str(v) for v in o]) + else: + return str(o) + + def generate_key(key): + return prefix + s(key).replace(" ", "") + + return generate_key + + +def create_cache(name): + ch = None + + try: + ch = get_region(name) + # The Goal evaluation Cache is implemented as a two-level cache (persistent in db, non-persistent as dogpile) + except: + ch = make_region().configure('dogpile.cache.memory') + warnings.warn("Warning: cache objects are in memory, are you creating docs?") + + ch.key_mangler = my_key_mangler(name) + + return ch + + + +cache_general = create_cache("general") +cache_achievement_eval = create_cache("achievement_eval") +cache_achievements_by_user_for_today = create_cache("achievements_by_user_for_today") +cache_achievements_users_levels = create_cache("achievements_users_levels") +cache_translations = create_cache("translations") +# The Goal evaluation Cache is implemented as a two-level cache (persistent in db, non-persistent as dogpile) +cache_goal_evaluation = create_cache("goal_evaluation") +cache_goal_statements = create_cache("goal_statements") + + +def clear_all_caches(): + cache_achievement_eval.invalidate(hard=True) + cache_achievements_by_user_for_today.invalidate(hard=True) + cache_translations.invalidate(hard=True) + cache_general.invalidate(hard=True) + invalidate_all_mc() + + +# URL Cache + +# -*- coding: utf-8 -*- +from pymemcache.client import Client + +host = "localhost" +port = 11211 +urlprefix = "" +is_active = True +urlcacheid = "gengine" + + +def setup_urlcache(prefix, url, active, id): + global urlprefix, host, port, is_active, urlcacheid + urlprefix = prefix + host, port = url.split(":") + port = int(port) + is_active = active + urlcacheid = id + + +def __build_key(key): + return "::URL_CACHE::" + str(urlcacheid) + "::" + urlprefix + str(key) + + +def get_or_set(key, generator): + if is_active: + client = Client((host, port)) + key = __build_key(key) + result = client.get(key) + if not result: + result = generator() + client.set(key, result) + client.quit() + return result + else: + return generator() + + +def set_value(key, value): + if is_active: + client = Client((host, port)) + key = __build_key(key) + client.set(key, value) + client.quit() + + +def invalidate(key): + if is_active: + key = __build_key(key) + client = Client((host, port)) + client.delete(key) + client.quit() + + +def invalidate_all_mc(): + if is_active: + client = Client((host, port)) + client.flush_all() + client.quit() \ No newline at end of file diff --git a/gengine/formular.py b/gengine/formular.py new file mode 100644 index 0000000..c08eca6 --- /dev/null +++ b/gengine/formular.py @@ -0,0 +1,67 @@ +import sys +from sqlalchemy.sql.expression import and_, or_ + +if sys.version_info < (3,5): + import __builtin__ +else: + import builtins as __builtin__ + +safe_list = ['math', 'acos', 'asin', 'atan', 'atan2', 'ceil', + 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', + 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', + 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'sum', 'range', 'str', 'int', 'float'] + +# use the list to filter the local namespace +from math import * + +safe_dict = dict([(k, locals().get(k, None)) for k in safe_list]) +for k in safe_dict.keys(): + if safe_dict[k] is None: + if hasattr(__builtin__, k): + safe_dict[k] = getattr(__builtin__, k) +safe_dict['and_'] = and_ +safe_dict['or_'] = or_ +safe_dict['abs'] = abs + +class FormularEvaluationException(Exception): + pass + +class DictObjectProxy(): + obj = None + + def __init__(self, obj): + self.obj = obj + + def __getattr__(self, name): + if not name in self.obj: + return "" + return self.obj[name] + +# TODO: Cache +def eval_formular(s, params={}): + """evaluates the formular. + + parameters are available as p.name, + + available math functions: + 'math','acos', 'asin', 'atan', 'atan2', 'ceil', + 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', + 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', + 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'sum', 'range' + """ + try: + if s is None: + return None + else: + p = DictObjectProxy(params) + + # add any needed builtins back in. + safe_dict['p'] = p + + result = eval(s, {"__builtins__": None}, safe_dict) + if type(result) == str: + return result % params + else: + return result + except: + raise FormularEvaluationException(s) \ No newline at end of file diff --git a/gengine/model_base.py b/gengine/model_base.py new file mode 100644 index 0000000..22f6900 --- /dev/null +++ b/gengine/model_base.py @@ -0,0 +1,90 @@ +import pytz +from pytz.exceptions import UnknownTimeZoneError +from sqlalchemy.sql.expression import select +from sqlalchemy.sql.functions import func +from zope.sqlalchemy.datamanager import mark_changed + +from gengine.cache import cache_general +from gengine.metadata import DBSession + + +class ABase(object): + """abstract base class which introduces a nice constructor for the model classes.""" + + def __init__(self, *args, **kw): + """ create a model object. + + pass attributes by using named parameters, e.g. name="foo", value=123 + """ + + for k, v in kw.items(): + setattr(self, k, v) + + def __str__(self): + if hasattr(self, "__unicode__"): + return self.__unicode__() + + +def calc_distance(latlong1, latlong2): + """generates a sqlalchemy expression for distance query in km + + :param latlong1: the location from which we look for rows, as tuple (lat,lng) + + :param latlong2: the columns containing the latitude and longitude, as tuple (lat,lng) + """ + + # explain: http://geokoder.com/distances + + # return func.sqrt(func.pow(69.1 * (latlong1[0] - latlong2[0]),2) + # + func.pow(53.0 * (latlong1[1] - latlong2[1]),2)) + + return func.sqrt(func.pow(111.2 * (latlong1[0] - latlong2[0]), 2) + + func.pow(111.2 * (latlong1[1] - latlong2[1]) * func.cos(latlong2[0]), 2)) + + +def coords(row): + return (row["lat"], row["lng"]) + + +def combine_updated_at(list_of_dates): + return max(list_of_dates) + + +def get_insert_id_by_result(r): + return r.last_inserted_ids()[0] + + +def get_insert_ids_by_result(r): + return r.last_inserted_ids() + + +def exists_by_expr(t, expr): + # TODO: use exists instead of count + q = select([func.count("*").label("c")], from_obj=t).where(expr) + r = DBSession.execute(q).fetchone() + if r.c > 0: + return True + else: + return False + + +@cache_general.cache_on_arguments() +def datetime_trunc(field, timezone): + return "date_trunc('%(field)s', CAST(to_char(NOW() AT TIME ZONE %(timezone)s, 'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP)) AT TIME ZONE %(timezone)s" % { + "field": field, + "timezone": timezone + } + + +@cache_general.cache_on_arguments() +def valid_timezone(timezone): + try: + pytz.timezone(timezone) + except UnknownTimeZoneError: + return False + return True + +def update_connection(): + session = DBSession() + mark_changed(session) + return session diff --git a/gengine/model_olymp.py b/gengine/model_olymp.py new file mode 100644 index 0000000..c20ee18 --- /dev/null +++ b/gengine/model_olymp.py @@ -0,0 +1,46 @@ +from alembic import command +from alembic.config import Config +from sqlalchemy import event +from sqlalchemy.orm import mapper +from sqlalchemy.sql.schema import Table, Column, MetaData +import sqlalchemy.types as ty + +from gengine.metadata import Base, DBSession +from gengine.model_base import ABase + +OLYMP_SCHEMA = "olymp" + +t_tenants = Table("tenants", Base.metadata, + Column("id", ty.String(), primary_key=True), + schema = OLYMP_SCHEMA +) + + +class Tenant(ABase): + def __unicode__(self, *args, **kwargs): + return "(ID: %s)" % (self.id,) + + +mapper(Tenant, t_tenants) + + +@event.listens_for(Tenant, "after_insert") +def create_tenant_schema(mapper, connection, target): + tenant_meta = MetaData(bind=connection) + schema = "t_"+target.id + + connection.execute("CREATE SCHEMA IF NOT EXISTS "+schema) + connection.execute("SET search_path TO "+schema) + + from . import model_tenant + + tables = [t.tometadata(tenant_meta, schema=schema) for name, t in model_tenant.__dict__.items() if isinstance(t, Table)] + + tenant_meta.create_all(tables=tables) + + alembic_cfg = Config(attributes={ + 'engine' : connection, + 'schema' : schema + }) + alembic_cfg.set_main_option("script_location", "gengine/alembic") + command.stamp(alembic_cfg, "head") \ No newline at end of file diff --git a/gengine/models.py b/gengine/model_tenant.py similarity index 90% rename from gengine/models.py rename to gengine/model_tenant.py index 600c7ce..c7698a0 100644 --- a/gengine/models.py +++ b/gengine/model_tenant.py @@ -7,10 +7,7 @@ import hashlib import pytz import sqlalchemy.types as ty -import warnings -from dogpile.cache import make_region -from pyramid_dogpile_cache import get_region -from pytz.exceptions import UnknownTimeZoneError + from sqlalchemy import ( Table, ForeignKey, @@ -28,53 +25,15 @@ ) from zope.sqlalchemy.datamanager import mark_changed +from gengine.cache import cache_general, cache_goal_evaluation, cache_achievement_eval, cache_achievements_users_levels, \ + cache_achievements_by_user_for_today, invalidate, cache_goal_statements, cache_translations from gengine.metadata import Base, DBSession -from . import urlcache - -try: - import __builtin__ -except: - #py35 - import builtins as __builtin__ -from sqlalchemy.sql import bindparam - -def my_key_mangler(prefix): - def s(o): - if type(o)==dict: - return "_".join(["%s=%s" % (str(k),str(v)) for k,v in o.items()]) - if type(o)==tuple: - return "_".join([str(v) for v in o]) - if type(o)==list: - return "_".join([str(v) for v in o]) - else: - return str(o) - - def generate_key(key): - return prefix + s(key).replace(" ","") +from gengine.model_base import exists_by_expr, datetime_trunc, calc_distance, coords, update_connection - return generate_key +from .model_base import ABase +from .formular import eval_formular -def create_cache(name): - ch = None - - try: - ch = get_region(name) - # The Goal evaluation Cache is implemented as a two-level cache (persistent in db, non-persistent as dogpile) - except: - ch = make_region().configure('dogpile.cache.memory') - warnings.warn("Warning: cache objects are in memory, are you creating docs?") - - ch.key_mangler = my_key_mangler(name) - globals()["cache_"+name] = ch - -create_cache("general") -create_cache("achievement_eval") -create_cache("achievements_by_user_for_today") -create_cache("achievements_users_levels") -create_cache("translations") -# The Goal evaluation Cache is implemented as a two-level cache (persistent in db, non-persistent as dogpile) -create_cache("goal_evaluation") -create_cache("goal_statements") +from sqlalchemy.sql import bindparam t_users = Table("users", Base.metadata, Column('id', ty.BigInteger, primary_key = True), @@ -235,22 +194,6 @@ def create_cache(name): Column('text', ty.Text(), nullable = False), ) -class ABase(object): - """abstract base class which introduces a nice constructor for the model classes.""" - - def __init__(self,*args,**kw): - """ create a model object. - - pass attributes by using named parameters, e.g. name="foo", value=123 - """ - - for k,v in kw.items(): - setattr(self, k, v) - - def __str__(self): - if hasattr(self, "__unicode__"): - return self.__unicode__() - class User(ABase): """A user participates in the gamification, i.e. can get achievements, rewards, participate in leaderbaord etc.""" @@ -753,7 +696,7 @@ def invalidate_evaluate_cache(cls,user_id,achievement): #We neeed to invalidate for all relevant users because of the leaderboards for uid in Achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user_id): cache_achievement_eval.delete("%s_%s" % (uid,achievement["id"])) - urlcache.invalidate("/progress/"+str(uid)) + invalidate("/progress/"+str(uid)) @classmethod @cache_general.cache_on_arguments() @@ -1281,126 +1224,4 @@ def insert_variable_for_property(mapper,connection,target): variable.group = "day" DBSession.add(variable) -#some query helpers - -def calc_distance(latlong1, latlong2): - """generates a sqlalchemy expression for distance query in km - - :param latlong1: the location from which we look for rows, as tuple (lat,lng) - - :param latlong2: the columns containing the latitude and longitude, as tuple (lat,lng) - """ - - #explain: http://geokoder.com/distances - - #return func.sqrt(func.pow(69.1 * (latlong1[0] - latlong2[0]),2) - # + func.pow(53.0 * (latlong1[1] - latlong2[1]),2)) - - return func.sqrt(func.pow(111.2 * (latlong1[0]-latlong2[0]),2) - + func.pow(111.2 * (latlong1[1]-latlong2[1]) * func.cos(latlong2[0]),2)) - -def coords(row): - return (row["lat"],row["lng"]) - -safe_list = ['math','acos', 'asin', 'atan', 'atan2', 'ceil', - 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', - 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', - 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'sum', 'range', 'str', 'int', 'float'] - -#use the list to filter the local namespace -from math import * -safe_dict = dict([ (k, locals().get(k, None)) for k in safe_list]) -for k in safe_dict.keys(): - if safe_dict[k] is None: - if hasattr(__builtin__, k): - safe_dict[k] = getattr(__builtin__, k) -safe_dict['and_'] = and_ -safe_dict['or_'] = or_ -safe_dict['abs'] = abs - -class FormularEvaluationException(Exception): - pass - -#TODO: Cache -def eval_formular(s,params={}): - """evaluates the formular. - - parameters are available as p.name, - - available math functions: - 'math','acos', 'asin', 'atan', 'atan2', 'ceil', - 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', - 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', - 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'sum', 'range' - """ - try: - if s is None: - return None - else: - p = DictObjectProxy(params) - - #add any needed builtins back in. - safe_dict['p'] = p - - result = eval(s,{"__builtins__":None},safe_dict) - if type(result)==str or type(result)==unicode: - return result % params - else: - return result - except: - raise FormularEvaluationException(s) - -class DictObjectProxy(): - obj = None - - def __init__(self, obj): - self.obj = obj - def __getattr__(self, name): - if not name in self.obj: - return "" - return self.obj[name] - -def combine_updated_at(list_of_dates): - return max(list_of_dates) - -def get_insert_id_by_result(r): - return r.last_inserted_ids()[0] - -def get_insert_ids_by_result(r): - return r.last_inserted_ids() - -def exists_by_expr(t, expr): - #TODO: use exists instead of count - q = select([func.count("*").label("c")], from_obj=t).where(expr) - r = DBSession.execute(q).fetchone() - if r.c > 0: - return True - else: - return False - -@cache_general.cache_on_arguments() -def datetime_trunc(field,timezone): - return "date_trunc('%(field)s', CAST(to_char(NOW() AT TIME ZONE %(timezone)s, 'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP)) AT TIME ZONE %(timezone)s" % { - "field" : field, - "timezone" : timezone - } - -@cache_general.cache_on_arguments() -def valid_timezone(timezone): - try: - pytz.timezone(timezone) - except UnknownTimeZoneError: - return False - return True - -def update_connection(): - session = DBSession() - mark_changed(session) - return session -def clear_all_caches(): - cache_achievement_eval.invalidate(hard=True) - cache_achievements_by_user_for_today.invalidate(hard=True) - cache_translations.invalidate(hard=True) - cache_general.invalidate(hard=True) - urlcache.invalidate_all() diff --git a/gengine/olympadmin.py b/gengine/olympadmin.py new file mode 100644 index 0000000..0cfaebf --- /dev/null +++ b/gengine/olympadmin.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +from flask import Flask +from flask.ext.admin import Admin +from flask.ext.admin.contrib.sqla import ModelView + +from gengine.metadata import DBSession +from gengine.model_olymp import Tenant +import pkg_resources, os +from flask.helpers import send_from_directory +import jinja2 + +olympadminapp = None +admin = None + +def resole_uri(uri): + from pyramid.path import PkgResourcesAssetDescriptor + pkg_name, path = uri.split(":", 1) + a = PkgResourcesAssetDescriptor(pkg_name, path) + absolute = a.abspath() # this is sometimes not absolute :-/ + absolute = os.path.abspath(absolute) # so we make it absolute + return absolute + + +def get_static_view(folder, olympadminapp): + folder = resole_uri(folder) + + def send_static_file(filename): + cache_timeout = olympadminapp.get_send_file_max_age(filename) + return send_from_directory(folder, filename, cache_timeout=cache_timeout) + + return send_static_file + + +def init_admin(urlprefix="", secret="fKY7kJ2xSrbPC5yieEjV", override_admin=None, override_olympadminapp=None): + global olympadminapp, admin + + if not override_olympadminapp: + olympadminapp = Flask(__name__) + olympadminapp.debug = True + olympadminapp.secret_key = secret + olympadminapp.config.update(dict( + PREFERRED_URL_SCHEME='https' + )) + else: + olympadminapp = override_olympadminapp + + # lets add our template directory + my_loader = jinja2.ChoiceLoader([ + olympadminapp.jinja_loader, + jinja2.FileSystemLoader(resole_uri("gengine:templates_olymp")), + ]) + olympadminapp.jinja_loader = my_loader + + olympadminapp.add_url_rule('/static_gengine/', + endpoint='static_gengine', + view_func=get_static_view('gengine:flask_static', olympadminapp)) + + @olympadminapp.context_processor + def inject_version(): + return {"gamification_engine_version": pkg_resources.get_distribution("gamification-engine").version} + + if not override_admin: + admin = Admin(olympadminapp, + name = "Zeus at Olympia - Gamification Engine", + base_template = 'admin_layout.html', + url = urlprefix + "/admin" + ) + else: + admin = override_admin + + admin.add_view(ModelViewTenant(DBSession)) + +class ModelViewTenant(ModelView): + column_list = ('id',) + column_searchable_list = ('id',) + form_columns = ('id',) + fast_mass_delete = True + + def __init__(self, session, **kwargs): + super(ModelViewTenant, self).__init__(Tenant, session, **kwargs) + diff --git a/gengine/resources.py b/gengine/resources.py new file mode 100644 index 0000000..5c3fe6c --- /dev/null +++ b/gengine/resources.py @@ -0,0 +1,32 @@ +from sqlalchemy.sql.expression import select +from gengine.metadata import DBSession +from gengine.model_olymp import t_tenants + +def root_factory(request): + return RootResource() + +class RootResource(): + def __getitem__(self, item): + DBSession.execute("SET search_path TO olymp") + if item=="t": + return TenantCollectionResource(self, "t") + raise KeyError() + +class TenantCollectionResource(): + def __init__(self, parent, name): + self.__name__ = name + self.__parent__ = parent + + def __getitem__(self, item): + q = select([t_tenants.c.id,], from_obj = t_tenants).where(t_tenants.c.id==item) + tenant = DBSession.execute(q).fetchone() + if tenant: + DBSession.execute("SET search_path TO t_%s" % tenant["id"]) + return TenantResource(self, tenant) + raise KeyError() + +class TenantResource(): + def __init__(self, parent, tenant): + self.__name__ = tenant["id"] + self.__parent__ = parent + self.model = tenant diff --git a/gengine/scripts/initializedb.py b/gengine/scripts/initializedb.py index 5a44ef2..c5fe27a 100644 --- a/gengine/scripts/initializedb.py +++ b/gengine/scripts/initializedb.py @@ -13,20 +13,22 @@ from pyramid.scripts.common import parse_vars import pyramid_dogpile_cache from pyramid.config import Configurator +from sqlalchemy.sql.schema import Table +from zope.sqlalchemy.datamanager import mark_changed + def usage(argv): cmd = os.path.basename(argv[0]) - print('usage: %s [var=value]\n' - '(example: "%s production.ini alembic.ini")' % (cmd, cmd)) + print('usage: %s [var=value]\n' + '(example: "%s production.ini")' % (cmd, cmd)) sys.exit(1) def main(argv=sys.argv): - if len(argv) < 3: + if len(argv) < 2: usage(argv) config_uri = argv[1] - alembic_uri = argv[2] - options = parse_vars(argv[3:]) + options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) @@ -43,21 +45,61 @@ def main(argv=sys.argv): config = Configurator(settings=settings) pyramid_dogpile_cache.includeme(config) - from ..metadata import ( + from gengine.metadata import ( init_session, - init_declarative_base + init_declarative_base, + init_db ) init_session() init_declarative_base() + init_db(engine) - from ..metadata import ( + from gengine.metadata import ( Base, DBSession ) - - from ..models import ( + + if options.get("reset_db",False): + Base.metadata.drop_all(engine) + engine.execute("DROP SCHEMA IF EXISTS olymp CASCADE") + + tenant_schemas = engine.execute("SELECT DISTINCT schemaname FROM pg_catalog.pg_tables WHERE schemaname LIKE 't_%%'") + + for t in tenant_schemas: + engine.execute("DROP SCHEMA %s CASCADE" % (t["schemaname"],)) + + engine.execute("CREATE SCHEMA IF NOT EXISTS olymp") + + from gengine import model_olymp + + tables = [t for name, t in model_olymp.__dict__.items() if isinstance(t, Table)] + Base.metadata.create_all(engine, tables=tables) + + # then, load the Alembic configuration and generate the + # version table, "stamping" it with the most recent rev: + from alembic.config import Config + from alembic import command + + alembic_cfg = Config(attributes={ + 'engine' : engine, + 'schema' : 'olymp' + }) + alembic_cfg.set_main_option("script_location", "gengine/alembic") + + command.stamp(alembic_cfg, "head") + + if options.get("populate_demo",False): + with transaction.manager: + t = model_olymp.Tenant(id="demo") + DBSession.add(t) + + populate_demo(DBSession, "demo") + +def populate_demo(DBSession, tenant_id): + + from gengine.model_tenant import ( Achievement, - AchievementCategory, + AchievementCategory, Goal, Variable, User, @@ -65,154 +107,140 @@ def main(argv=sys.argv): TranslationVariable, Translation, GoalProperty, - GoalGoalProperty, + GoalGoalProperty, Reward, AchievementProperty, - AchievementAchievementProperty, + AchievementAchievementProperty, AchievementReward ) - - DBSession.configure(bind=engine) - - if options.get("reset_db",False): - Base.metadata.drop_all(engine) - - Base.metadata.create_all(engine) - - - # then, load the Alembic configuration and generate the - # version table, "stamping" it with the most recent rev: - from alembic.config import Config - from alembic import command - alembic_cfg = Config(alembic_uri) - command.stamp(alembic_cfg, "head") - - - + def add_translation_variable(name): t = TranslationVariable(name=name) DBSession.add(t) return t - - def add_translation(variable,lang,text): - tr = Translation(translationvariable=variable,text=text,language=lang) + + def add_translation(variable, lang, text): + tr = Translation(translationvariable=variable, text=text, language=lang) DBSession.add(tr) return tr - - if options.get("populate_demo",False): - with transaction.manager: - - lang_de = Language(name="de") - lang_en = Language(name="en") - DBSession.add(lang_de) - DBSession.add(lang_en) - - var_invited_users = Variable(name="invite_users") - DBSession.add(var_invited_users) - - var_invited_users = Variable(name="participate", - group="none") - DBSession.add(var_invited_users) - - goal_property_name = GoalProperty(name='name') - DBSession.add(goal_property_name) - - achievementcategory_community = AchievementCategory(name="community") - DBSession.add(achievementcategory_community) - - achievement_invite = Achievement(name='invite_users', - evaluation="immediately", - maxtimes=20, - achievementcategory=achievementcategory_community) - DBSession.add(achievement_invite) - - transvar_invite = add_translation_variable(name="invite_users_goal_name") - add_translation(transvar_invite, lang_en, '"Invite "+`(5*p.level)`+" Users"') - add_translation(transvar_invite, lang_de, '"Lade "+`(5*p.level)`+" Freunde ein"') - - achievement_invite_goal1 = Goal(name_translation=transvar_invite, - condition='p.var=="invite_users"', - goal="5*p.level", - operator="geq", - achievement=achievement_invite) - DBSession.add(achievement_invite_goal1) - - DBSession.add(GoalGoalProperty(goal=achievement_invite_goal1, property=goal_property_name, value_translation=transvar_invite)) - - achievementcategory_sports = AchievementCategory(name="sports") - DBSession.add(achievementcategory_sports) - - achievement_fittest = Achievement(name='fittest', - relevance="friends", - maxlevel=100, - achievementcategory=achievementcategory_sports) - DBSession.add(achievement_fittest) - - transvar_fittest = add_translation_variable(name="fittest_goal_name") - add_translation(transvar_fittest, lang_en, '"Do the most sport activities among your friends"') - add_translation(transvar_fittest, lang_de, '"Mache unter deinen Freunden am meisten Sportaktivitäten"') - - achievement_fittest_goal1 = Goal(name_translation=transvar_fittest, - condition='and_(p.var=="participate", p.key.in_(["5","7","9"]))', - evaluation="weekly", - goal="5*p.level", - achievement=achievement_fittest - ) - - DBSession.add(achievement_fittest_goal1) - DBSession.add(GoalGoalProperty(goal=achievement_fittest_goal1, property=goal_property_name, value_translation=transvar_fittest)) - - property_name = AchievementProperty(name='name') - DBSession.add(property_name) - - property_xp = AchievementProperty(name='xp') - DBSession.add(property_xp) - - property_icon = AchievementProperty(name='icon') - DBSession.add(property_icon) - - reward_badge = Reward(name='badge') - DBSession.add(reward_badge) - - reward_image = Reward(name='backgroud_image') - DBSession.add(reward_image) - - transvar_invite_name = add_translation_variable(name="invite_achievement_name") - add_translation(transvar_invite_name, lang_en, '"The Community!"') - add_translation(transvar_invite_name, lang_de, '"Die Community!"') - - DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_name, value_translation=transvar_invite_name)) - DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_xp, value='100 * p.level')) - DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_icon, value="'https://www.gamification-software.com/img/running.png'")) - - DBSession.add(AchievementReward(achievement=achievement_invite, reward=reward_badge, value="'https://www.gamification-software.com/img/trophy.png'", from_level=5)) - DBSession.add(AchievementReward(achievement=achievement_invite, reward=reward_image, value="'https://www.gamification-software.com/img/video-controller-336657_1920.jpg'", from_level=5)) - - transvar_fittest_name = add_translation_variable(name="fittest_achievement_name") - add_translation(transvar_fittest_name, lang_en, '"The Fittest!"') - add_translation(transvar_fittest_name, lang_de, '"Der Fitteste!"') - - DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_name, value_translation=transvar_fittest_name)) - DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_xp, value='50 + (200 * p.level)')) - DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_icon, value="'https://www.gamification-software.com/img/colorwheel.png'")) - - DBSession.add(AchievementReward(achievement=achievement_fittest, reward=reward_badge, value="'https://www.gamification-software.com/img/easel.png'", from_level=1)) - DBSession.add(AchievementReward(achievement=achievement_fittest, reward=reward_image, value="'https://www.gamification-software.com/img/game-characters-622654.jpg'", from_level=1)) - - - user1 = User(id=1,lat=10,lng=50,timezone="Europe/Berlin") - user2 = User(id=2,lat=10,lng=50,timezone="US/Eastern") - user3 = User(id=3,lat=10,lng=50) - - user1.friends.append(user2) - user1.friends.append(user3) - - user2.friends.append(user1) - user2.friends.append(user3) - - user3.friends.append(user1) - user3.friends.append(user2) - - DBSession.add(user1) - DBSession.add(user2) - DBSession.add(user3) \ No newline at end of file + + with transaction.manager: + DBSession.execute("SET search_path TO t_%s" % tenant_id) + + lang_de = Language(name="de") + lang_en = Language(name="en") + DBSession.add(lang_de) + DBSession.add(lang_en) + + var_invited_users = Variable(name="invite_users") + DBSession.add(var_invited_users) + + var_invited_users = Variable(name="participate", + group="none") + DBSession.add(var_invited_users) + + goal_property_name = GoalProperty(name='name') + DBSession.add(goal_property_name) + + achievementcategory_community = AchievementCategory(name="community") + DBSession.add(achievementcategory_community) + + achievement_invite = Achievement(name='invite_users', + evaluation="immediately", + maxtimes=20, + achievementcategory=achievementcategory_community) + DBSession.add(achievement_invite) + + transvar_invite = add_translation_variable(name="invite_users_goal_name") + add_translation(transvar_invite, lang_en, '"Invite "+`(5*p.level)`+" Users"') + add_translation(transvar_invite, lang_de, '"Lade "+`(5*p.level)`+" Freunde ein"') + + achievement_invite_goal1 = Goal(name_translation=transvar_invite, + condition='p.var=="invite_users"', + goal="5*p.level", + operator="geq", + achievement=achievement_invite) + DBSession.add(achievement_invite_goal1) + + DBSession.add(GoalGoalProperty(goal=achievement_invite_goal1, property=goal_property_name, value_translation=transvar_invite)) + + achievementcategory_sports = AchievementCategory(name="sports") + DBSession.add(achievementcategory_sports) + + achievement_fittest = Achievement(name='fittest', + relevance="friends", + maxlevel=100, + achievementcategory=achievementcategory_sports) + DBSession.add(achievement_fittest) + + transvar_fittest = add_translation_variable(name="fittest_goal_name") + add_translation(transvar_fittest, lang_en, '"Do the most sport activities among your friends"') + add_translation(transvar_fittest, lang_de, '"Mache unter deinen Freunden am meisten Sportaktivitäten"') + + achievement_fittest_goal1 = Goal(name_translation=transvar_fittest, + condition='and_(p.var=="participate", p.key.in_(["5","7","9"]))', + evaluation="weekly", + goal="5*p.level", + achievement=achievement_fittest + ) + + DBSession.add(achievement_fittest_goal1) + DBSession.add(GoalGoalProperty(goal=achievement_fittest_goal1, property=goal_property_name, value_translation=transvar_fittest)) + + property_name = AchievementProperty(name='name') + DBSession.add(property_name) + + property_xp = AchievementProperty(name='xp') + DBSession.add(property_xp) + + property_icon = AchievementProperty(name='icon') + DBSession.add(property_icon) + + reward_badge = Reward(name='badge') + DBSession.add(reward_badge) + + reward_image = Reward(name='backgroud_image') + DBSession.add(reward_image) + + transvar_invite_name = add_translation_variable(name="invite_achievement_name") + add_translation(transvar_invite_name, lang_en, '"The Community!"') + add_translation(transvar_invite_name, lang_de, '"Die Community!"') + + DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_name, value_translation=transvar_invite_name)) + DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_xp, value='100 * p.level')) + DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_icon, value="'https://www.gamification-software.com/img/running.png'")) + + DBSession.add(AchievementReward(achievement=achievement_invite, reward=reward_badge, value="'https://www.gamification-software.com/img/trophy.png'", from_level=5)) + DBSession.add(AchievementReward(achievement=achievement_invite, reward=reward_image, value="'https://www.gamification-software.com/img/video-controller-336657_1920.jpg'", from_level=5)) + + transvar_fittest_name = add_translation_variable(name="fittest_achievement_name") + add_translation(transvar_fittest_name, lang_en, '"The Fittest!"') + add_translation(transvar_fittest_name, lang_de, '"Der Fitteste!"') + + DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_name, value_translation=transvar_fittest_name)) + DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_xp, value='50 + (200 * p.level)')) + DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_icon, value="'https://www.gamification-software.com/img/colorwheel.png'")) + + DBSession.add(AchievementReward(achievement=achievement_fittest, reward=reward_badge, value="'https://www.gamification-software.com/img/easel.png'", from_level=1)) + DBSession.add(AchievementReward(achievement=achievement_fittest, reward=reward_image, value="'https://www.gamification-software.com/img/game-characters-622654.jpg'", from_level=1)) + + + user1 = User(id=1,lat=10,lng=50,timezone="Europe/Berlin") + user2 = User(id=2,lat=10,lng=50,timezone="US/Eastern") + user3 = User(id=3,lat=10,lng=50) + + user1.friends.append(user2) + user1.friends.append(user3) + + user2.friends.append(user1) + user2.friends.append(user3) + + user3.friends.append(user1) + user3.friends.append(user2) + + DBSession.add(user1) + DBSession.add(user2) + DBSession.add(user3) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/gengine/templates_olymp/admin/index.html b/gengine/templates_olymp/admin/index.html new file mode 100644 index 0000000..194abf6 --- /dev/null +++ b/gengine/templates_olymp/admin/index.html @@ -0,0 +1,37 @@ +{% extends 'admin/master.html' %} +{% import 'admin/lib.html' as lib with context %} + +{% macro extra() %} + +{% endmacro %} + +{% block head %} + {{ super() }} +{% endblock %} + +{% block body %} +
    + {% for msg in msgs %} +
  • {{ msg }}
    • + {% endfor %} +
+ +

Global Administration ( Olymp )

+ +
+ +
+ +
+
+ +
+
+
+ +
+{% endblock %} + +{% block tail %} + {{ super() }} +{% endblock %} diff --git a/gengine/templates/admin/layout.html b/gengine/templates_olymp/admin/layout.html similarity index 100% rename from gengine/templates/admin/layout.html rename to gengine/templates_olymp/admin/layout.html diff --git a/gengine/templates_olymp/admin_layout.html b/gengine/templates_olymp/admin_layout.html new file mode 100644 index 0000000..95acd9d --- /dev/null +++ b/gengine/templates_olymp/admin_layout.html @@ -0,0 +1,12 @@ +{% import 'admin/layout.html' as layout with context -%} +{% extends 'admin/base.html' %} + +{% block head_tail %} + {{ super() }} + +{% endblock %} + +{% block tail %} + {{ super() }} + +{% endblock %} diff --git a/gengine/templates/error.html b/gengine/templates_olymp/error.html similarity index 100% rename from gengine/templates/error.html rename to gengine/templates_olymp/error.html diff --git a/gengine/templates/admin/index.html b/gengine/templates_tenant/admin/index.html similarity index 100% rename from gengine/templates/admin/index.html rename to gengine/templates_tenant/admin/index.html diff --git a/gengine/templates_tenant/admin/layout.html b/gengine/templates_tenant/admin/layout.html new file mode 100644 index 0000000..12e3f9a --- /dev/null +++ b/gengine/templates_tenant/admin/layout.html @@ -0,0 +1,81 @@ +{% macro menu_icon(item) -%} +{% set icon_type = item.get_icon_type() %} +{%- if icon_type %} + {% set icon_value = item.get_icon_value() %} + {% if icon_type == 'glyph' %} + + {% elif icon_type == 'image' %} + menu image + {% elif icon_type == 'image-url' %} + menu image + {% endif %} +{% endif %} +{%- endmacro %} + +{% macro menu() %} + {%- for item in admin_view.admin.menu() %} + {%- if item.is_category() -%} + {% set children = item.get_children() %} + {%- if children %} + {% set class_name = item.get_class_name() %} + {%- if item.is_active(admin_view) %} +
  • + {% else -%} +
  • + {%- endif %} + {{ menu_icon(item) }}{{ item.name }} +
      + {%- for child in children -%} + {% set class_name = child.get_class_name() %} + {%- if child.is_active(admin_view) %} +
    • + {% else %} + + {%- endif %} + {{ menu_icon(child) }}{{ child.name }} +
      • + {%- endfor %} +
    +
    • + {% endif %} + {%- else %} + {%- if item.is_accessible() and item.is_visible() -%} + {% set class_name = item.get_class_name() %} + {%- if item.is_active(admin_view) %} +
  • + {%- else %} + + {%- endif %} + {{ menu_icon(item) }}{{ item.name }} +
    • + {%- endif -%} + {% endif -%} + {% endfor %} +{% endmacro %} + +{% macro menu_links() %} + {% for item in admin_view.admin.menu_links() %} + {% if item.is_accessible() and item.is_visible() %} +
  • + {{ menu_icon(item) }}{{ item.name }} +
    • + {% endif %} + {% endfor %} +{% endmacro %} + +{% macro messages() %} + {% with messages = get_flashed_messages(with_categories=True) %} + {% if messages %} + {% for category, m in messages %} + {% if category %} +
    + {% else %} +
    + {% endif %} + x + {{ m }} +
    + {% endfor %} + {% endif %} + {% endwith %} +{% endmacro %} diff --git a/gengine/templates/admin_layout.html b/gengine/templates_tenant/admin_layout.html similarity index 94% rename from gengine/templates/admin_layout.html rename to gengine/templates_tenant/admin_layout.html index f9bdff9..96ff752 100644 --- a/gengine/templates/admin_layout.html +++ b/gengine/templates_tenant/admin_layout.html @@ -8,7 +8,7 @@ {% block tail %} {{ super() }} - + {% endblock %} diff --git a/gengine/templates/admin_maintenance.html b/gengine/templates_tenant/admin_maintenance.html similarity index 100% rename from gengine/templates/admin_maintenance.html rename to gengine/templates_tenant/admin_maintenance.html diff --git a/gengine/templates_tenant/error.html b/gengine/templates_tenant/error.html new file mode 100644 index 0000000..380d49c --- /dev/null +++ b/gengine/templates_tenant/error.html @@ -0,0 +1,20 @@ + + + + + Error + + +
    +
    +

    + + ${message} +

    +

    + ${description} +

    +
    +
    + + diff --git a/gengine/flaskadmin.py b/gengine/tenantadmin.py similarity index 87% rename from gengine/flaskadmin.py rename to gengine/tenantadmin.py index 65ec996..40d92d6 100644 --- a/gengine/flaskadmin.py +++ b/gengine/tenantadmin.py @@ -3,7 +3,7 @@ from flask.ext.admin import Admin from flask.ext.admin.contrib.sqla import ModelView -from gengine.models import DBSession, Variable, Goal, AchievementCategory, Achievement, AchievementProperty, GoalProperty, AchievementAchievementProperty, AchievementReward,\ +from .model_tenant import DBSession, Variable, Goal, AchievementCategory, Achievement, AchievementProperty, GoalProperty, AchievementAchievementProperty, AchievementReward,\ GoalGoalProperty, Reward, User, GoalEvaluationCache, Value, AchievementUser, TranslationVariable, Language, Translation from flask_admin.contrib.sqla.filters import BooleanEqualFilter, IntEqualFilter from flask_admin.base import AdminIndexView, BaseView, expose @@ -15,7 +15,7 @@ import jinja2 from flask_admin.model.form import InlineFormAdmin -flaskadminapp=None +tenantadminapp=None admin=None @@ -36,40 +36,40 @@ def send_static_file(filename): return send_static_file -def init_flaskadmin(urlprefix="",secret="fKY7kJ2xSrbPC5yieEjV",override_admin=None,override_flaskadminapp=None): - global flaskadminapp, admin +def init_admin(urlprefix="",secret="fKY7kJ2xSrbPC5yieEjV",override_admin=None,override_flaskadminapp=None): + global tenantadminapp, admin if not override_flaskadminapp: - flaskadminapp = Flask(__name__) - flaskadminapp.debug=True - flaskadminapp.secret_key = secret - flaskadminapp.config.update(dict( + tenantadminapp = Flask(__name__) + tenantadminapp.debug=True + tenantadminapp.secret_key = secret + tenantadminapp.config.update(dict( PREFERRED_URL_SCHEME = 'https' )) else: - flaskadminapp = override_flaskadminapp - - # lets add our template directory - my_loader = jinja2.ChoiceLoader([ - flaskadminapp.jinja_loader, - jinja2.FileSystemLoader(resole_uri("gengine:templates")), - ]) - flaskadminapp.jinja_loader = my_loader + tenantadminapp = override_flaskadminapp + + # lets add our template directory + my_loader = jinja2.ChoiceLoader([ + tenantadminapp.jinja_loader, + jinja2.FileSystemLoader(resole_uri("gengine:templates")), + ]) + tenantadminapp.jinja_loader = my_loader - flaskadminapp.add_url_rule('/static_gengine/', - endpoint='static_gengine', - view_func=get_static_view('gengine:flask_static',flaskadminapp)) + tenantadminapp.add_url_rule('/static_gengine/', + endpoint='static_gengine', + view_func=get_static_view('gengine:flask_static', tenantadminapp)) - @flaskadminapp.context_processor + @tenantadminapp.context_processor def inject_version(): return { "gamification_engine_version" : pkg_resources.get_distribution("gamification-engine").version } if not override_admin: - admin = Admin(flaskadminapp, + admin = Admin(tenantadminapp, name="Gamification Engine - Admin Control Panel", base_template='admin_layout.html', url=urlprefix+"/admin" - ) + ) else: admin = override_admin diff --git a/gengine/urlcache.py b/gengine/urlcache.py deleted file mode 100644 index a36371f..0000000 --- a/gengine/urlcache.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -from pymemcache.client import Client - -host = "localhost" -port = 11211 -urlprefix = "" -is_active = True -urlcacheid = "gengine" - -def setup_urlcache(prefix, url, active, id): - global urlprefix, host, port, is_active, urlcacheid - urlprefix = prefix - host, port = url.split(":") - port = int(port) - is_active = active - urlcacheid = id - -def __build_key(key): - return "::URL_CACHE::"+str(urlcacheid)+"::"+urlprefix+str(key) - -def get_or_set(key,generator): - if is_active: - client = Client((host,port)) - key = __build_key(key) - result = client.get(key) - if not result: - result = generator() - client.set(key, result) - client.quit() - return result - else: - return generator() - -def set_value(key,value): - if is_active: - client = Client((host,port)) - key = __build_key(key) - client.set(key, value) - client.quit() - -def invalidate(key): - if is_active: - key = __build_key(key) - client = Client((host,port)) - client.delete(key) - client.quit() - -def invalidate_all(): - if is_active: - client = Client((host,port)) - client.flush_all() - client.quit() \ No newline at end of file diff --git a/gengine/views.py b/gengine/views.py index cdff0eb..b7d17b4 100644 --- a/gengine/views.py +++ b/gengine/views.py @@ -1,25 +1,26 @@ # -*- coding: utf-8 -*- -from pyramid.view import view_config -from .models import ( - User, - Achievement, - Value - ) - -from .urlcache import get_or_set -from pyramid.renderers import render, JSON +import traceback -from .flaskadmin import flaskadminapp -from pyramid.wsgi import wsgiapp2, wsgiapp -from _collections import defaultdict -from gengine.models import Variable, valid_timezone, Goal, AchievementReward, FormularEvaluationException -from gengine.urlcache import set_value -from pyramid.exceptions import NotFound, HTTPBadRequest +from gengine.cache import set_value +from pyramid.exceptions import NotFound +from pyramid.renderers import render +from pyramid.view import view_config +from pyramid.wsgi import wsgiapp2 from werkzeug import DebuggedApplication + +from gengine.formular import FormularEvaluationException +from gengine.model_base import valid_timezone from gengine.wsgiutil import HTTPSProxied +from .cache import get_or_set from .errors import APIError - -import traceback +from .model_tenant import ( + User, + Achievement, + Value, +Variable +) +from .tenantadmin import tenantadminapp +from .olympadmin import olympadminapp @view_config(route_name='add_or_update_user', renderer='string', request_method="POST") def add_or_update_user(request): @@ -236,8 +237,12 @@ def generate(): return get_or_set(key,generate) -@view_config(route_name='admin') +@view_config(route_name='admin_tenant') +@wsgiapp2 +def admin_tenant(environ, start_response): + return HTTPSProxied(DebuggedApplication(tenantadminapp.wsgi_app, True))(environ, start_response) + +@view_config(route_name='admin_olymp') @wsgiapp2 -def admin(environ, start_response): - return HTTPSProxied(DebuggedApplication(flaskadminapp.wsgi_app, True))(environ,start_response) - #return \ No newline at end of file +def admin_olymp(environ, start_response): + return HTTPSProxied(DebuggedApplication(olympadminapp.wsgi_app, True))(environ, start_response) From 466048ed2498ff7a466a5ce8ca0f77098f7a8038 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Sat, 11 Jun 2016 18:57:29 +0200 Subject: [PATCH 002/176] a lot refactoring --- gengine/__init__.py | 30 ++++----- gengine/base/__init__.py | 0 gengine/{ => base}/cache.py | 0 gengine/{ => base}/errors.py | 0 gengine/{model_base.py => base/model.py} | 2 +- gengine/maintenance/__init__.py | 0 gengine/{ => maintenance}/scripts/__init__.py | 0 .../{ => maintenance}/scripts/generate_erd.py | 0 .../{ => maintenance}/scripts/initializedb.py | 23 +++---- .../{ => maintenance}/scripts/quickstart.py | 0 gengine/olymp/__init__.py | 0 gengine/{olympadmin.py => olymp/admin.py} | 13 ++-- gengine/{ => olymp}/alembic/README | 0 gengine/{ => olymp}/alembic/env.py | 6 +- gengine/{ => olymp}/alembic/script.py.mako | 0 gengine/{model_olymp.py => olymp/model.py} | 10 +-- gengine/olymp/route.py | 3 + .../templates}/admin/index.html | 0 .../templates}/admin/layout.html | 0 .../templates}/admin_layout.html | 0 .../templates}/error.html | 0 gengine/olymp/views.py | 14 ++++ gengine/resources.py | 4 +- gengine/tenant/__init__.py | 0 gengine/{tenantadmin.py => tenant/admin.py} | 23 +++---- gengine/tenant/alembic/README | 1 + gengine/tenant/alembic/env.py | 66 +++++++++++++++++++ gengine/tenant/alembic/script.py.mako | 24 +++++++ gengine/{ => tenant}/formular.py | 0 gengine/{model_tenant.py => tenant/model.py} | 15 ++--- gengine/tenant/route.py | 9 +++ .../{flask_static => tenant/static}/admin.js | 0 .../static}/admin_layout.css | 0 .../{flask_static => tenant/static}/api.js | 2 +- .../templates}/admin/index.html | 0 .../templates}/admin/layout.html | 0 .../templates}/admin_layout.html | 5 +- .../templates}/admin_maintenance.html | 0 .../templates}/error.html | 0 gengine/{ => tenant}/views.py | 24 +++---- 40 files changed, 189 insertions(+), 85 deletions(-) create mode 100644 gengine/base/__init__.py rename gengine/{ => base}/cache.py (100%) rename gengine/{ => base}/errors.py (100%) rename gengine/{model_base.py => base/model.py} (98%) create mode 100644 gengine/maintenance/__init__.py rename gengine/{ => maintenance}/scripts/__init__.py (100%) rename gengine/{ => maintenance}/scripts/generate_erd.py (100%) rename gengine/{ => maintenance}/scripts/initializedb.py (96%) rename gengine/{ => maintenance}/scripts/quickstart.py (100%) create mode 100644 gengine/olymp/__init__.py rename gengine/{olympadmin.py => olymp/admin.py} (93%) rename gengine/{ => olymp}/alembic/README (100%) rename gengine/{ => olymp}/alembic/env.py (94%) rename gengine/{ => olymp}/alembic/script.py.mako (100%) rename gengine/{model_olymp.py => olymp/model.py} (84%) create mode 100644 gengine/olymp/route.py rename gengine/{templates_olymp => olymp/templates}/admin/index.html (100%) rename gengine/{templates_olymp => olymp/templates}/admin/layout.html (100%) rename gengine/{templates_olymp => olymp/templates}/admin_layout.html (100%) rename gengine/{templates_olymp => olymp/templates}/error.html (100%) create mode 100644 gengine/olymp/views.py create mode 100644 gengine/tenant/__init__.py rename gengine/{tenantadmin.py => tenant/admin.py} (95%) create mode 100644 gengine/tenant/alembic/README create mode 100644 gengine/tenant/alembic/env.py create mode 100644 gengine/tenant/alembic/script.py.mako rename gengine/{ => tenant}/formular.py (100%) rename gengine/{model_tenant.py => tenant/model.py} (99%) create mode 100644 gengine/tenant/route.py rename gengine/{flask_static => tenant/static}/admin.js (100%) rename gengine/{flask_static => tenant/static}/admin_layout.css (100%) rename gengine/{flask_static => tenant/static}/api.js (98%) rename gengine/{templates_tenant => tenant/templates}/admin/index.html (100%) rename gengine/{templates_tenant => tenant/templates}/admin/layout.html (100%) rename gengine/{templates_tenant => tenant/templates}/admin_layout.html (84%) rename gengine/{templates_tenant => tenant/templates}/admin_maintenance.html (100%) rename gengine/{templates_tenant => tenant/templates}/error.html (100%) rename gengine/{ => tenant}/views.py (94%) diff --git a/gengine/__init__.py b/gengine/__init__.py index 2aa9758..6687f46 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -1,18 +1,18 @@ # -*- coding: utf-8 -*- -from pyramid import events __version__ = '0.1.36' -import datetime, os +import datetime +import os from pyramid.config import Configurator from pyramid.renderers import JSON - +from pyramid.settings import asbool from sqlalchemy import engine_from_config -from pyramid.settings import asbool from gengine.wsgiutil import HTTPSProxied, init_reverse_proxy + def main(global_config, **settings): """ This function returns a Pyramid WSGI application. """ @@ -49,28 +49,24 @@ def main(global_config, **settings): urlcache_active = asbool(os.environ.get("URLCACHE_ACTIVE", settings.get("urlcache_active",True))) #routes - config.add_route('get_progress', urlprefix+'/t/{tenant}/progress/{user_id}', traverse="/t/{tenant}") - config.add_route('increase_value', urlprefix+'/t/{tenant}/increase_value/{variable_name}/{user_id}', traverse="/t/{tenant}") - config.add_route('increase_value_with_key', urlprefix+'/t/{tenant}/increase_value/{variable_name}/{user_id}/{key}', traverse="/t/{tenant}") - config.add_route('increase_multi_values', urlprefix+'/t/{tenant}/increase_multi_values', traverse="/t/{tenant}") - config.add_route('add_or_update_user', urlprefix+'/t/{tenant}/add_or_update_user/{user_id}', traverse="/t/{tenant}") - config.add_route('delete_user', urlprefix+'/t/{tenant}/delete_user/{user_id}', traverse="/t/{tenant}") - config.add_route('get_achievement_level', urlprefix+'/t/{tenant}/achievement/{achievement_id}/level/{level}', traverse="/t/{tenant}") - #config.add_route('get_achievement_reward', urlprefix+'/achievement_reward/{achievement_reward_id}') - - config.add_route('admin_tenant', '/t/{tenant}/*subpath', traverse="/t/{tenant}") #prefix is set in flaskadmin.py + from gengine.tenant.route import config_routes as config_tenant_routes + from gengine.olymp.route import config_routes as config_olymp_routes + config.include(config_tenant_routes, route_prefix=urlprefix) + config.include(config_olymp_routes, route_prefix=urlprefix) + + config.add_route('admin_tenant', '/t/{tenant}/*subpath', traverse="/t/{tenant}") #prefix is set in flaskadmin.py config.add_route('admin_olymp', '/olymp/*subpath') # prefix is set in flaskadmin.py - from gengine.tenantadmin import init_admin as init_tenantadmin + from gengine.tenant.admin import init_admin as init_tenantadmin init_tenantadmin(urlprefix=urlprefix, secret=settings.get("flaskadmin_secret","fKY7kJ2xSrbPC5yieEjV")) - from gengine.olympadmin import init_admin as init_olympadmin + from gengine.olymp.admin import init_admin as init_olympadmin init_olympadmin(urlprefix=urlprefix, secret=settings.get("flaskadmin_secret", "fKY7kJ2xSrbPC5yieEjV")) - from .cache import setup_urlcache + from gengine.base.cache import setup_urlcache setup_urlcache(prefix=urlprefix, url = urlcache_url, active = urlcache_active, diff --git a/gengine/base/__init__.py b/gengine/base/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gengine/cache.py b/gengine/base/cache.py similarity index 100% rename from gengine/cache.py rename to gengine/base/cache.py diff --git a/gengine/errors.py b/gengine/base/errors.py similarity index 100% rename from gengine/errors.py rename to gengine/base/errors.py diff --git a/gengine/model_base.py b/gengine/base/model.py similarity index 98% rename from gengine/model_base.py rename to gengine/base/model.py index 22f6900..bbf3853 100644 --- a/gengine/model_base.py +++ b/gengine/base/model.py @@ -4,8 +4,8 @@ from sqlalchemy.sql.functions import func from zope.sqlalchemy.datamanager import mark_changed -from gengine.cache import cache_general from gengine.metadata import DBSession +from gengine.base.cache import cache_general class ABase(object): diff --git a/gengine/maintenance/__init__.py b/gengine/maintenance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gengine/scripts/__init__.py b/gengine/maintenance/scripts/__init__.py similarity index 100% rename from gengine/scripts/__init__.py rename to gengine/maintenance/scripts/__init__.py diff --git a/gengine/scripts/generate_erd.py b/gengine/maintenance/scripts/generate_erd.py similarity index 100% rename from gengine/scripts/generate_erd.py rename to gengine/maintenance/scripts/generate_erd.py diff --git a/gengine/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py similarity index 96% rename from gengine/scripts/initializedb.py rename to gengine/maintenance/scripts/initializedb.py index c5fe27a..9f93f00 100644 --- a/gengine/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -1,20 +1,17 @@ # -*- coding: utf-8 -*- -import os import sys -import transaction - -from sqlalchemy import engine_from_config +import os +import pyramid_dogpile_cache +import transaction +from pyramid.config import Configurator from pyramid.paster import ( get_appsettings, setup_logging, ) - from pyramid.scripts.common import parse_vars -import pyramid_dogpile_cache -from pyramid.config import Configurator +from sqlalchemy import engine_from_config from sqlalchemy.sql.schema import Table -from zope.sqlalchemy.datamanager import mark_changed def usage(argv): @@ -70,9 +67,9 @@ def main(argv=sys.argv): engine.execute("CREATE SCHEMA IF NOT EXISTS olymp") - from gengine import model_olymp + from gengine.olymp import model - tables = [t for name, t in model_olymp.__dict__.items() if isinstance(t, Table)] + tables = [t for name, t in model.__dict__.items() if isinstance(t, Table)] Base.metadata.create_all(engine, tables=tables) # then, load the Alembic configuration and generate the @@ -84,20 +81,20 @@ def main(argv=sys.argv): 'engine' : engine, 'schema' : 'olymp' }) - alembic_cfg.set_main_option("script_location", "gengine/alembic") + alembic_cfg.set_main_option("script_location", "gengine/olymp/alembic") command.stamp(alembic_cfg, "head") if options.get("populate_demo",False): with transaction.manager: - t = model_olymp.Tenant(id="demo") + t = model.Tenant(id="demo") DBSession.add(t) populate_demo(DBSession, "demo") def populate_demo(DBSession, tenant_id): - from gengine.model_tenant import ( + from gengine.tenant.model import ( Achievement, AchievementCategory, Goal, diff --git a/gengine/scripts/quickstart.py b/gengine/maintenance/scripts/quickstart.py similarity index 100% rename from gengine/scripts/quickstart.py rename to gengine/maintenance/scripts/quickstart.py diff --git a/gengine/olymp/__init__.py b/gengine/olymp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gengine/olympadmin.py b/gengine/olymp/admin.py similarity index 93% rename from gengine/olympadmin.py rename to gengine/olymp/admin.py index 0cfaebf..f6360e0 100644 --- a/gengine/olympadmin.py +++ b/gengine/olymp/admin.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- +import jinja2 +import os +import pkg_resources from flask import Flask from flask.ext.admin import Admin from flask.ext.admin.contrib.sqla import ModelView +from flask.helpers import send_from_directory from gengine.metadata import DBSession -from gengine.model_olymp import Tenant -import pkg_resources, os -from flask.helpers import send_from_directory -import jinja2 +from gengine.olymp.model import Tenant olympadminapp = None admin = None @@ -47,13 +48,13 @@ def init_admin(urlprefix="", secret="fKY7kJ2xSrbPC5yieEjV", override_admin=None, # lets add our template directory my_loader = jinja2.ChoiceLoader([ olympadminapp.jinja_loader, - jinja2.FileSystemLoader(resole_uri("gengine:templates_olymp")), + jinja2.FileSystemLoader(resole_uri("gengine:olymp/templates")), ]) olympadminapp.jinja_loader = my_loader olympadminapp.add_url_rule('/static_gengine/', endpoint='static_gengine', - view_func=get_static_view('gengine:flask_static', olympadminapp)) + view_func=get_static_view('gengine:olymp/static', olympadminapp)) @olympadminapp.context_processor def inject_version(): diff --git a/gengine/alembic/README b/gengine/olymp/alembic/README similarity index 100% rename from gengine/alembic/README rename to gengine/olymp/alembic/README diff --git a/gengine/alembic/env.py b/gengine/olymp/alembic/env.py similarity index 94% rename from gengine/alembic/env.py rename to gengine/olymp/alembic/env.py index 861893a..25332d4 100644 --- a/gengine/alembic/env.py +++ b/gengine/olymp/alembic/env.py @@ -1,14 +1,12 @@ from __future__ import with_statement -from alembic import context -from sqlalchemy import engine_from_config, pool + from logging.config import fileConfig import os - +from alembic import context # this is the Alembic Config object, which provides # access to the values within the .ini file in use. -from sqlalchemy.engine import create_engine config = context.config diff --git a/gengine/alembic/script.py.mako b/gengine/olymp/alembic/script.py.mako similarity index 100% rename from gengine/alembic/script.py.mako rename to gengine/olymp/alembic/script.py.mako diff --git a/gengine/model_olymp.py b/gengine/olymp/model.py similarity index 84% rename from gengine/model_olymp.py rename to gengine/olymp/model.py index c20ee18..3c7a9a6 100644 --- a/gengine/model_olymp.py +++ b/gengine/olymp/model.py @@ -1,12 +1,12 @@ +import sqlalchemy.types as ty from alembic import command from alembic.config import Config from sqlalchemy import event from sqlalchemy.orm import mapper from sqlalchemy.sql.schema import Table, Column, MetaData -import sqlalchemy.types as ty -from gengine.metadata import Base, DBSession -from gengine.model_base import ABase +from gengine.metadata import Base +from gengine.base.model import ABase OLYMP_SCHEMA = "olymp" @@ -32,7 +32,7 @@ def create_tenant_schema(mapper, connection, target): connection.execute("CREATE SCHEMA IF NOT EXISTS "+schema) connection.execute("SET search_path TO "+schema) - from . import model_tenant + from gengine.tenant import model as model_tenant tables = [t.tometadata(tenant_meta, schema=schema) for name, t in model_tenant.__dict__.items() if isinstance(t, Table)] @@ -42,5 +42,5 @@ def create_tenant_schema(mapper, connection, target): 'engine' : connection, 'schema' : schema }) - alembic_cfg.set_main_option("script_location", "gengine/alembic") + alembic_cfg.set_main_option("script_location", "gengine/tenant/alembic") command.stamp(alembic_cfg, "head") \ No newline at end of file diff --git a/gengine/olymp/route.py b/gengine/olymp/route.py new file mode 100644 index 0000000..3a65854 --- /dev/null +++ b/gengine/olymp/route.py @@ -0,0 +1,3 @@ + +def config_routes(config): + pass \ No newline at end of file diff --git a/gengine/templates_olymp/admin/index.html b/gengine/olymp/templates/admin/index.html similarity index 100% rename from gengine/templates_olymp/admin/index.html rename to gengine/olymp/templates/admin/index.html diff --git a/gengine/templates_olymp/admin/layout.html b/gengine/olymp/templates/admin/layout.html similarity index 100% rename from gengine/templates_olymp/admin/layout.html rename to gengine/olymp/templates/admin/layout.html diff --git a/gengine/templates_olymp/admin_layout.html b/gengine/olymp/templates/admin_layout.html similarity index 100% rename from gengine/templates_olymp/admin_layout.html rename to gengine/olymp/templates/admin_layout.html diff --git a/gengine/templates_olymp/error.html b/gengine/olymp/templates/error.html similarity index 100% rename from gengine/templates_olymp/error.html rename to gengine/olymp/templates/error.html diff --git a/gengine/olymp/views.py b/gengine/olymp/views.py new file mode 100644 index 0000000..c13af1c --- /dev/null +++ b/gengine/olymp/views.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +from pyramid.view import view_config +from pyramid.wsgi import wsgiapp2 +from werkzeug import DebuggedApplication + +from gengine.olymp.admin import olympadminapp + +from gengine.wsgiutil import HTTPSProxied + +@view_config(route_name='admin_olymp') +@wsgiapp2 +def admin_olymp(environ, start_response): + return HTTPSProxied(DebuggedApplication(olympadminapp.wsgi_app, True))(environ, start_response) diff --git a/gengine/resources.py b/gengine/resources.py index 5c3fe6c..b2d7579 100644 --- a/gengine/resources.py +++ b/gengine/resources.py @@ -1,6 +1,8 @@ from sqlalchemy.sql.expression import select + from gengine.metadata import DBSession -from gengine.model_olymp import t_tenants +from gengine.olymp.model import t_tenants + def root_factory(request): return RootResource() diff --git a/gengine/tenant/__init__.py b/gengine/tenant/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gengine/tenantadmin.py b/gengine/tenant/admin.py similarity index 95% rename from gengine/tenantadmin.py rename to gengine/tenant/admin.py index 40d92d6..5cef712 100644 --- a/gengine/tenantadmin.py +++ b/gengine/tenant/admin.py @@ -1,19 +1,20 @@ # -*- coding: utf-8 -*- +import jinja2 +import os +import pkg_resources from flask import Flask from flask.ext.admin import Admin from flask.ext.admin.contrib.sqla import ModelView - -from .model_tenant import DBSession, Variable, Goal, AchievementCategory, Achievement, AchievementProperty, GoalProperty, AchievementAchievementProperty, AchievementReward,\ - GoalGoalProperty, Reward, User, GoalEvaluationCache, Value, AchievementUser, TranslationVariable, Language, Translation -from flask_admin.contrib.sqla.filters import BooleanEqualFilter, IntEqualFilter -from flask_admin.base import AdminIndexView, BaseView, expose -from wtforms import BooleanField from flask.globals import request -from wtforms.form import Form -import pkg_resources, os from flask.helpers import send_from_directory -import jinja2 +from flask_admin.base import BaseView, expose +from flask_admin.contrib.sqla.filters import IntEqualFilter from flask_admin.model.form import InlineFormAdmin +from wtforms import BooleanField +from wtforms.form import Form + +from gengine.tenant.model import DBSession, Variable, Goal, AchievementCategory, Achievement, AchievementProperty, GoalProperty, AchievementAchievementProperty, AchievementReward,\ + GoalGoalProperty, Reward, User, GoalEvaluationCache, Value, AchievementUser, TranslationVariable, Language, Translation tenantadminapp=None admin=None @@ -52,13 +53,13 @@ def init_admin(urlprefix="",secret="fKY7kJ2xSrbPC5yieEjV",override_admin=None,ov # lets add our template directory my_loader = jinja2.ChoiceLoader([ tenantadminapp.jinja_loader, - jinja2.FileSystemLoader(resole_uri("gengine:templates")), + jinja2.FileSystemLoader(resole_uri("gengine:tenant/templates")), ]) tenantadminapp.jinja_loader = my_loader tenantadminapp.add_url_rule('/static_gengine/', endpoint='static_gengine', - view_func=get_static_view('gengine:flask_static', tenantadminapp)) + view_func=get_static_view('gengine:tenant/static', tenantadminapp)) @tenantadminapp.context_processor def inject_version(): diff --git a/gengine/tenant/alembic/README b/gengine/tenant/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/gengine/tenant/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/gengine/tenant/alembic/env.py b/gengine/tenant/alembic/env.py new file mode 100644 index 0000000..25332d4 --- /dev/null +++ b/gengine/tenant/alembic/env.py @@ -0,0 +1,66 @@ +from __future__ import with_statement + +from logging.config import fileConfig + +import os +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. + +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name: + fileConfig(config.config_file_name) + +overrides = {} + +durl = os.environ.get("DATABASE_URL") #heroku +if durl: + config.set_main_option('sqlalchemy.url',durl) + +# add your model's MetaData object here +# for 'autogenerate' support +from gengine.metadata import init_session,init_declarative_base +init_session() +init_declarative_base() + +from gengine.metadata import Base +target_metadata = Base.metadata + +# target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + engine = config.attributes["engine"] + + schema = config.attributes["schema"] + + #connectable = create_engine(url, poolclass=pool.NullPool) + + with engine.connect() as connection: + + connection.execute("SET search_path TO "+schema) + + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +run_migrations_online() diff --git a/gengine/tenant/alembic/script.py.mako b/gengine/tenant/alembic/script.py.mako new file mode 100644 index 0000000..43c0940 --- /dev/null +++ b/gengine/tenant/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/gengine/formular.py b/gengine/tenant/formular.py similarity index 100% rename from gengine/formular.py rename to gengine/tenant/formular.py diff --git a/gengine/model_tenant.py b/gengine/tenant/model.py similarity index 99% rename from gengine/model_tenant.py rename to gengine/tenant/model.py index c7698a0..35ace0a 100644 --- a/gengine/model_tenant.py +++ b/gengine/tenant/model.py @@ -7,7 +7,9 @@ import hashlib import pytz import sqlalchemy.types as ty - +from gengine.base.model import ABase, exists_by_expr, datetime_trunc, calc_distance, coords, update_connection +from gengine.base.cache import cache_general, cache_goal_evaluation, cache_achievement_eval, cache_achievements_users_levels, \ + cache_achievements_by_user_for_today, invalidate, cache_goal_statements, cache_translations from sqlalchemy import ( Table, ForeignKey, @@ -23,17 +25,10 @@ mapper, relationship ) -from zope.sqlalchemy.datamanager import mark_changed +from sqlalchemy.sql import bindparam -from gengine.cache import cache_general, cache_goal_evaluation, cache_achievement_eval, cache_achievements_users_levels, \ - cache_achievements_by_user_for_today, invalidate, cache_goal_statements, cache_translations from gengine.metadata import Base, DBSession -from gengine.model_base import exists_by_expr, datetime_trunc, calc_distance, coords, update_connection - -from .model_base import ABase -from .formular import eval_formular - -from sqlalchemy.sql import bindparam +from gengine.tenant.formular import eval_formular t_users = Table("users", Base.metadata, Column('id', ty.BigInteger, primary_key = True), diff --git a/gengine/tenant/route.py b/gengine/tenant/route.py new file mode 100644 index 0000000..9138adf --- /dev/null +++ b/gengine/tenant/route.py @@ -0,0 +1,9 @@ + +def config_routes(config): + config.add_route('get_progress', '/t/{tenant}/progress/{user_id}', traverse="/t/{tenant}") + config.add_route('increase_value', '/t/{tenant}/increase_value/{variable_name}/{user_id}', traverse="/t/{tenant}") + config.add_route('increase_value_with_key', '/t/{tenant}/increase_value/{variable_name}/{user_id}/{key}', traverse="/t/{tenant}") + config.add_route('increase_multi_values', '/t/{tenant}/increase_multi_values', traverse="/t/{tenant}") + config.add_route('add_or_update_user', '/t/{tenant}/add_or_update_user/{user_id}', traverse="/t/{tenant}") + config.add_route('delete_user', '/t/{tenant}/delete_user/{user_id}', traverse="/t/{tenant}") + config.add_route('get_achievement_level', '/t/{tenant}/achievement/{achievement_id}/level/{level}', traverse="/t/{tenant}") \ No newline at end of file diff --git a/gengine/flask_static/admin.js b/gengine/tenant/static/admin.js similarity index 100% rename from gengine/flask_static/admin.js rename to gengine/tenant/static/admin.js diff --git a/gengine/flask_static/admin_layout.css b/gengine/tenant/static/admin_layout.css similarity index 100% rename from gengine/flask_static/admin_layout.css rename to gengine/tenant/static/admin_layout.css diff --git a/gengine/flask_static/api.js b/gengine/tenant/static/api.js similarity index 98% rename from gengine/flask_static/api.js rename to gengine/tenant/static/api.js index 5d2f56e..85fcb26 100644 --- a/gengine/flask_static/api.js +++ b/gengine/tenant/static/api.js @@ -40,7 +40,7 @@ var setupAPIForm = function($, defaultcall, fields, api_funcs) { var api_settings_postparams; var setURL = function(url,method,postparams) { - api_settings_url = url; + api_settings_url = API_BASE_URL ? API_BASE_URL+url : url; api_settings_method=method; api_settings_postparams = postparams; }; diff --git a/gengine/templates_tenant/admin/index.html b/gengine/tenant/templates/admin/index.html similarity index 100% rename from gengine/templates_tenant/admin/index.html rename to gengine/tenant/templates/admin/index.html diff --git a/gengine/templates_tenant/admin/layout.html b/gengine/tenant/templates/admin/layout.html similarity index 100% rename from gengine/templates_tenant/admin/layout.html rename to gengine/tenant/templates/admin/layout.html diff --git a/gengine/templates_tenant/admin_layout.html b/gengine/tenant/templates/admin_layout.html similarity index 84% rename from gengine/templates_tenant/admin_layout.html rename to gengine/tenant/templates/admin_layout.html index 96ff752..9e064e1 100644 --- a/gengine/templates_tenant/admin_layout.html +++ b/gengine/tenant/templates/admin_layout.html @@ -3,7 +3,10 @@ {% block head_tail %} {{ super() }} - + + {% endblock %} {% block tail %} diff --git a/gengine/templates_tenant/admin_maintenance.html b/gengine/tenant/templates/admin_maintenance.html similarity index 100% rename from gengine/templates_tenant/admin_maintenance.html rename to gengine/tenant/templates/admin_maintenance.html diff --git a/gengine/templates_tenant/error.html b/gengine/tenant/templates/error.html similarity index 100% rename from gengine/templates_tenant/error.html rename to gengine/tenant/templates/error.html diff --git a/gengine/views.py b/gengine/tenant/views.py similarity index 94% rename from gengine/views.py rename to gengine/tenant/views.py index b7d17b4..cd39236 100644 --- a/gengine/views.py +++ b/gengine/tenant/views.py @@ -1,26 +1,25 @@ # -*- coding: utf-8 -*- import traceback -from gengine.cache import set_value +from gengine.base.model import valid_timezone +from gengine.base.cache import get_or_set, set_value +from gengine.base.errors import APIError from pyramid.exceptions import NotFound from pyramid.renderers import render from pyramid.view import view_config from pyramid.wsgi import wsgiapp2 from werkzeug import DebuggedApplication -from gengine.formular import FormularEvaluationException -from gengine.model_base import valid_timezone -from gengine.wsgiutil import HTTPSProxied -from .cache import get_or_set -from .errors import APIError -from .model_tenant import ( +from gengine.olymp.admin import olympadminapp +from gengine.tenant.admin import tenantadminapp +from gengine.tenant.formular import FormularEvaluationException +from gengine.tenant.model import ( User, Achievement, Value, -Variable + Variable ) -from .tenantadmin import tenantadminapp -from .olympadmin import olympadminapp +from gengine.wsgiutil import HTTPSProxied @view_config(route_name='add_or_update_user', renderer='string', request_method="POST") def add_or_update_user(request): @@ -241,8 +240,3 @@ def generate(): @wsgiapp2 def admin_tenant(environ, start_response): return HTTPSProxied(DebuggedApplication(tenantadminapp.wsgi_app, True))(environ, start_response) - -@view_config(route_name='admin_olymp') -@wsgiapp2 -def admin_olymp(environ, start_response): - return HTTPSProxied(DebuggedApplication(olympadminapp.wsgi_app, True))(environ, start_response) From 7deeddf8a480227c7d7542680bbb7493197d8a47 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Sat, 11 Jun 2016 18:57:45 +0200 Subject: [PATCH 003/176] fix script paths --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index b007ddc..2796607 100644 --- a/setup.py +++ b/setup.py @@ -65,8 +65,8 @@ [paste.app_factory] main = gengine:main [console_scripts] - initialize_gengine_db = gengine.scripts.initializedb:main - gengine_quickstart = gengine.scripts.quickstart:main - generate_gengine_erd = gengine.scripts.generate_erd:main + initialize_gengine_db = gengine.maintenance.scripts.initializedb:main + gengine_quickstart = gengine.maintenance.scripts.quickstart:main + generate_gengine_erd = gengine.maintenance.scripts.generate_erd:main """, ) From ea0afd374fac3b24c3f78260dac316ef88913397 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 12 Jun 2016 16:15:11 +0200 Subject: [PATCH 004/176] version directory --- gengine/olymp/alembic/versions/DUMMY | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 gengine/olymp/alembic/versions/DUMMY diff --git a/gengine/olymp/alembic/versions/DUMMY b/gengine/olymp/alembic/versions/DUMMY new file mode 100644 index 0000000..e69de29 From e989ae9abd991afaeaad54c80acd273f3e27da2b Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 12 Jun 2016 16:15:52 +0200 Subject: [PATCH 005/176] version directory --- gengine/olymp/alembic/versions/DUMMY | 1 + 1 file changed, 1 insertion(+) diff --git a/gengine/olymp/alembic/versions/DUMMY b/gengine/olymp/alembic/versions/DUMMY index e69de29..90a1d60 100644 --- a/gengine/olymp/alembic/versions/DUMMY +++ b/gengine/olymp/alembic/versions/DUMMY @@ -0,0 +1 @@ +... \ No newline at end of file From fe62efc2f722a451fc59f89632d4364b169b6f14 Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 12 Jun 2016 16:17:39 +0200 Subject: [PATCH 006/176] version directory --- gengine/tenant/alembic/versions/DUMMY | 1 + 1 file changed, 1 insertion(+) create mode 100644 gengine/tenant/alembic/versions/DUMMY diff --git a/gengine/tenant/alembic/versions/DUMMY b/gengine/tenant/alembic/versions/DUMMY new file mode 100644 index 0000000..90a1d60 --- /dev/null +++ b/gengine/tenant/alembic/versions/DUMMY @@ -0,0 +1 @@ +... \ No newline at end of file From e1c0aae64863487c8b4c241cbe807b282526425f Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 12 Jun 2016 16:17:55 +0200 Subject: [PATCH 007/176] fix py 3 < 3.5 bultin import --- gengine/tenant/formular.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/tenant/formular.py b/gengine/tenant/formular.py index c08eca6..bb415e0 100644 --- a/gengine/tenant/formular.py +++ b/gengine/tenant/formular.py @@ -1,7 +1,7 @@ import sys from sqlalchemy.sql.expression import and_, or_ -if sys.version_info < (3,5): +if sys.version_info < (3,0): import __builtin__ else: import builtins as __builtin__ From 94c4adb271dc66ca88f9ce147618cddbcae243af Mon Sep 17 00:00:00 2001 From: Marcel Date: Sun, 12 Jun 2016 17:03:58 +0200 Subject: [PATCH 008/176] start 0.2.0 version --- gengine/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/__init__.py b/gengine/__init__.py index 6687f46..b9c0bc5 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '0.1.36' +__version__ = '0.2.0' import datetime From 019a7fac9eeb037ea63e992a5d24caf9a3f97a2f Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 13 Jun 2016 09:38:38 +0200 Subject: [PATCH 009/176] first draft of a condition term schema --- gengine/tenant/condition_term.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 gengine/tenant/condition_term.py diff --git a/gengine/tenant/condition_term.py b/gengine/tenant/condition_term.py new file mode 100644 index 0000000..e69de29 From 0d6c4c9360858b06172df4cb3255b432ab38d75c Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 13 Jun 2016 09:38:46 +0200 Subject: [PATCH 010/176] first draft of a condition term schema --- gengine/tenant/condition_term.py | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/gengine/tenant/condition_term.py b/gengine/tenant/condition_term.py index e69de29..0d88f52 100644 --- a/gengine/tenant/condition_term.py +++ b/gengine/tenant/condition_term.py @@ -0,0 +1,41 @@ +import jsl +import jsonschema + + +class Conjunction(jsl.Document): + terms = jsl.ArrayField(jsl.DocumentField("Term", as_ref=True), required=True) + type = jsl.StringField(pattern="^and$") + + +class Disjunction(jsl.Document): + terms = jsl.ArrayField(jsl.DocumentField("Term", as_ref=True), required=True) + type = jsl.StringField(pattern="^or$") + + +class Literal(jsl.Document): + variable = jsl.StringField(required=True) + key = jsl.ArrayField(jsl.StringField(), required=False) + + +class Term(jsl.Document): + + content = jsl.OneOfField([ + jsl.DocumentField(Conjunction, as_ref=True), + jsl.DocumentField(Disjunction, as_ref=True), + jsl.DocumentField(Literal, as_ref=True) + ], required=True) + + +def __get_schema(): + return Term.get_schema() + + +def validate_term(json): + return jsonschema.validate(json,__get_schema()) + +demo_schema = { + 'content' : { + 'variable' : 'participate', + 'key' : '2' + } +} From 749991df8e86abb626dce61c96a86397629c5154 Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 13 Jun 2016 09:43:13 +0200 Subject: [PATCH 011/176] rename type to operator, fix demo data --- gengine/tenant/condition_term.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gengine/tenant/condition_term.py b/gengine/tenant/condition_term.py index 0d88f52..f298236 100644 --- a/gengine/tenant/condition_term.py +++ b/gengine/tenant/condition_term.py @@ -4,12 +4,12 @@ class Conjunction(jsl.Document): terms = jsl.ArrayField(jsl.DocumentField("Term", as_ref=True), required=True) - type = jsl.StringField(pattern="^and$") + operator = jsl.StringField(pattern="^and$") class Disjunction(jsl.Document): terms = jsl.ArrayField(jsl.DocumentField("Term", as_ref=True), required=True) - type = jsl.StringField(pattern="^or$") + operator = jsl.StringField(pattern="^or$") class Literal(jsl.Document): @@ -33,9 +33,10 @@ def __get_schema(): def validate_term(json): return jsonschema.validate(json,__get_schema()) + demo_schema = { 'content' : { 'variable' : 'participate', - 'key' : '2' + 'key' : ['2',] } } From 456700ff1d59611ed675eaab2573e01e92b660d6 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Tue, 14 Jun 2016 10:28:49 +0200 Subject: [PATCH 012/176] fix a caching bug: Somehow dogpile seems to store the object lazily and now when calling. When we modify it afterwards, the modified version is sometimes cached. We solve this by doing a deepcopy before the modification. --- gengine/tenant/views.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/gengine/tenant/views.py b/gengine/tenant/views.py index cd39236..9b1f05c 100644 --- a/gengine/tenant/views.py +++ b/gengine/tenant/views.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import traceback +import copy + from gengine.base.model import valid_timezone from gengine.base.cache import get_or_set, set_value from gengine.base.errors import APIError @@ -121,7 +123,7 @@ def ea(achievement): set_value(key,ret_str) return ret_str, ret -@view_config(route_name='get_progress', renderer='json') +@view_config(route_name='get_progress', renderer='string') def get_progress(request): """get all relevant data concerning the user's progress""" user_id = int(request.matchdict["user_id"]) @@ -129,8 +131,12 @@ def get_progress(request): user = User.get_user(user_id) if not user: raise NotFound("user not found") - - return _get_progress(user, force_generation=False)[0] + + request.response.content_type = "application/json" + progress = _get_progress(user, force_generation=False) + json_string, pmap = progress + return json_string + @view_config(route_name='increase_value', renderer='json', request_method="POST") @view_config(route_name='increase_value_with_key', renderer='json', request_method="POST") @@ -156,13 +162,15 @@ def increase_value(request): Value.increase_value(variable_name, user, value, key) - output = _get_progress(user,force_generation=True)[1] - - for aid in output["achievements"].keys(): + output = copy.deepcopy(_get_progress(user,force_generation=True)[1]) #1 is the map + for aid in list(output["achievements"].keys()): if len(output["achievements"][aid]["new_levels"])>0: - del output["achievements"][aid]["levels"] - del output["achievements"][aid]["priority"] - del output["achievements"][aid]["goals"] + if "levels" in output["achievements"][aid]: + del output["achievements"][aid]["levels"] + if "priority" in output["achievements"][aid]: + del output["achievements"][aid]["priority"] + if "goals" in output["achievements"][aid]: + del output["achievements"][aid]["goals"] else: del output["achievements"][aid] return output @@ -194,16 +202,19 @@ def increase_multi_values(request): Value.increase_value(variable_name, user, value, key) - output = _get_progress(user,force_generation=True)[1] + output = copy.deepcopy(_get_progress(user, force_generation=True)[1]) # 1 is the map - for aid in output["achievements"].keys(): + for aid in list(output["achievements"].keys()): if len(output["achievements"][aid]["new_levels"])>0: - del output["achievements"][aid]["levels"] - del output["achievements"][aid]["priority"] - del output["achievements"][aid]["goals"] + if "levels" in output["achievements"][aid]: + del output["achievements"][aid]["levels"] + if "priority" in output["achievements"][aid]: + del output["achievements"][aid]["priority"] + if "goals" in output["achievements"][aid]: + del output["achievements"][aid]["goals"] else: del output["achievements"][aid] - + if len(output["achievements"])>0 : ret[user_id]=output From dbba018b51c875fa90dca76cf5dd7e0215b692fb Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Tue, 14 Jun 2016 10:31:45 +0200 Subject: [PATCH 013/176] replace ruling based on python evaluation by - a json schema for the conditions - a mathematical expression parser for the computations (goals) - a minimal string templating language for the computation of names --- gengine/base/cache.py | 2 + gengine/maintenance/scripts/initializedb.py | 40 +- gengine/tenant/admin.py | 2 +- gengine/tenant/condition_term.py | 42 -- gengine/tenant/formular.py | 309 +++++++++++--- gengine/tenant/model.py | 420 ++++++++++---------- gengine/tenant/views.py | 2 +- 7 files changed, 491 insertions(+), 326 deletions(-) delete mode 100644 gengine/tenant/condition_term.py diff --git a/gengine/base/cache.py b/gengine/base/cache.py index 3cc921c..7b2925c 100644 --- a/gengine/base/cache.py +++ b/gengine/base/cache.py @@ -51,6 +51,8 @@ def clear_all_caches(): cache_achievements_by_user_for_today.invalidate(hard=True) cache_translations.invalidate(hard=True) cache_general.invalidate(hard=True) + cache_goal_evaluation.invalidate(hard=True) + cache_goal_statements.invalidate(hard=True) invalidate_all_mc() diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index 9f93f00..155d7de 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -149,12 +149,12 @@ def add_translation(variable, lang, text): DBSession.add(achievement_invite) transvar_invite = add_translation_variable(name="invite_users_goal_name") - add_translation(transvar_invite, lang_en, '"Invite "+`(5*p.level)`+" Users"') - add_translation(transvar_invite, lang_de, '"Lade "+`(5*p.level)`+" Freunde ein"') + add_translation(transvar_invite, lang_en, 'Invite ${5*level} Users') + add_translation(transvar_invite, lang_de, 'Lade ${5*level} Freunde ein') achievement_invite_goal1 = Goal(name_translation=transvar_invite, - condition='p.var=="invite_users"', - goal="5*p.level", + condition='{"term": {"type": "literal", "variable": "invite_users"}}', + goal="5*level", operator="geq", achievement=achievement_invite) DBSession.add(achievement_invite_goal1) @@ -171,13 +171,13 @@ def add_translation(variable, lang, text): DBSession.add(achievement_fittest) transvar_fittest = add_translation_variable(name="fittest_goal_name") - add_translation(transvar_fittest, lang_en, '"Do the most sport activities among your friends"') - add_translation(transvar_fittest, lang_de, '"Mache unter deinen Freunden am meisten Sportaktivitäten"') + add_translation(transvar_fittest, lang_en, 'Do the most sport activities among your friends') + add_translation(transvar_fittest, lang_de, 'Mache unter deinen Freunden am meisten Sportaktivitäten') achievement_fittest_goal1 = Goal(name_translation=transvar_fittest, - condition='and_(p.var=="participate", p.key.in_(["5","7","9"]))', + condition='{"term": {"key": ["5","7","9"], "type": "literal", "key_operator": "IN", "variable": "participate"}}', evaluation="weekly", - goal="5*p.level", + goal="5*level", achievement=achievement_fittest ) @@ -200,26 +200,26 @@ def add_translation(variable, lang, text): DBSession.add(reward_image) transvar_invite_name = add_translation_variable(name="invite_achievement_name") - add_translation(transvar_invite_name, lang_en, '"The Community!"') - add_translation(transvar_invite_name, lang_de, '"Die Community!"') + add_translation(transvar_invite_name, lang_en, 'The Community!') + add_translation(transvar_invite_name, lang_de, 'Die Community!') DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_name, value_translation=transvar_invite_name)) - DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_xp, value='100 * p.level')) - DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_icon, value="'https://www.gamification-software.com/img/running.png'")) + DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_xp, value='${100 * level}')) + DBSession.add(AchievementAchievementProperty(achievement=achievement_invite, property=property_icon, value="https://www.gamification-software.com/img/running.png")) - DBSession.add(AchievementReward(achievement=achievement_invite, reward=reward_badge, value="'https://www.gamification-software.com/img/trophy.png'", from_level=5)) - DBSession.add(AchievementReward(achievement=achievement_invite, reward=reward_image, value="'https://www.gamification-software.com/img/video-controller-336657_1920.jpg'", from_level=5)) + DBSession.add(AchievementReward(achievement=achievement_invite, reward=reward_badge, value="https://www.gamification-software.com/img/trophy.png", from_level=5)) + DBSession.add(AchievementReward(achievement=achievement_invite, reward=reward_image, value="https://www.gamification-software.com/img/video-controller-336657_1920.jpg", from_level=5)) transvar_fittest_name = add_translation_variable(name="fittest_achievement_name") - add_translation(transvar_fittest_name, lang_en, '"The Fittest!"') - add_translation(transvar_fittest_name, lang_de, '"Der Fitteste!"') + add_translation(transvar_fittest_name, lang_en, 'The Fittest!') + add_translation(transvar_fittest_name, lang_de, 'Der Fitteste!') DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_name, value_translation=transvar_fittest_name)) - DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_xp, value='50 + (200 * p.level)')) - DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_icon, value="'https://www.gamification-software.com/img/colorwheel.png'")) + DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_xp, value='${50 + (200 * level)}')) + DBSession.add(AchievementAchievementProperty(achievement=achievement_fittest, property=property_icon, value="https://www.gamification-software.com/img/colorwheel.png")) - DBSession.add(AchievementReward(achievement=achievement_fittest, reward=reward_badge, value="'https://www.gamification-software.com/img/easel.png'", from_level=1)) - DBSession.add(AchievementReward(achievement=achievement_fittest, reward=reward_image, value="'https://www.gamification-software.com/img/game-characters-622654.jpg'", from_level=1)) + DBSession.add(AchievementReward(achievement=achievement_fittest, reward=reward_badge, value="https://www.gamification-software.com/img/easel.png", from_level=1)) + DBSession.add(AchievementReward(achievement=achievement_fittest, reward=reward_image, value="https://www.gamification-software.com/img/game-characters-622654.jpg", from_level=1)) user1 = User(id=1,lat=10,lng=50,timezone="Europe/Berlin") diff --git a/gengine/tenant/admin.py b/gengine/tenant/admin.py index 5cef712..1f46d2a 100644 --- a/gengine/tenant/admin.py +++ b/gengine/tenant/admin.py @@ -212,7 +212,7 @@ def index(self): self._template_args['clear_caches_form'] = self.clear_caches_form = ClearCacheForm(request.form) if request.method == 'POST': - from models import clear_all_caches + from gengine.base.cache import clear_all_caches if self.clear_caches_form.clear_check.data: clear_all_caches() self._template_args['msgs'].append("All caches cleared!") diff --git a/gengine/tenant/condition_term.py b/gengine/tenant/condition_term.py deleted file mode 100644 index f298236..0000000 --- a/gengine/tenant/condition_term.py +++ /dev/null @@ -1,42 +0,0 @@ -import jsl -import jsonschema - - -class Conjunction(jsl.Document): - terms = jsl.ArrayField(jsl.DocumentField("Term", as_ref=True), required=True) - operator = jsl.StringField(pattern="^and$") - - -class Disjunction(jsl.Document): - terms = jsl.ArrayField(jsl.DocumentField("Term", as_ref=True), required=True) - operator = jsl.StringField(pattern="^or$") - - -class Literal(jsl.Document): - variable = jsl.StringField(required=True) - key = jsl.ArrayField(jsl.StringField(), required=False) - - -class Term(jsl.Document): - - content = jsl.OneOfField([ - jsl.DocumentField(Conjunction, as_ref=True), - jsl.DocumentField(Disjunction, as_ref=True), - jsl.DocumentField(Literal, as_ref=True) - ], required=True) - - -def __get_schema(): - return Term.get_schema() - - -def validate_term(json): - return jsonschema.validate(json,__get_schema()) - - -demo_schema = { - 'content' : { - 'variable' : 'participate', - 'key' : ['2',] - } -} diff --git a/gengine/tenant/formular.py b/gengine/tenant/formular.py index bb415e0..643a64e 100644 --- a/gengine/tenant/formular.py +++ b/gengine/tenant/formular.py @@ -1,67 +1,270 @@ -import sys -from sqlalchemy.sql.expression import and_, or_ - -if sys.version_info < (3,0): - import __builtin__ -else: - import builtins as __builtin__ - -safe_list = ['math', 'acos', 'asin', 'atan', 'atan2', 'ceil', - 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', - 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', - 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'sum', 'range', 'str', 'int', 'float'] - -# use the list to filter the local namespace -from math import * - -safe_dict = dict([(k, locals().get(k, None)) for k in safe_list]) -for k in safe_dict.keys(): - if safe_dict[k] is None: - if hasattr(__builtin__, k): - safe_dict[k] = getattr(__builtin__, k) -safe_dict['and_'] = and_ -safe_dict['or_'] = or_ -safe_dict['abs'] = abs +import re + +import functools +import jsl +import json +import jsonschema +import pyparsing as pp + +import math +import operator + +from sqlalchemy.sql import and_, or_ class FormularEvaluationException(Exception): - pass + def __init__(self, message): + self.message = message + +# The Expression Parser for mathematical formulars +class NumericStringParser(object): + ''' + Most of this code comes from the fourFn.py pyparsing example + + # from: http://stackoverflow.com/a/2371789 -class DictObjectProxy(): - obj = None + ''' - def __init__(self, obj): - self.obj = obj + def pushFirst(self, strg, loc, toks): + self.exprStack.append(toks[0]) - def __getattr__(self, name): - if not name in self.obj: - return "" - return self.obj[name] + def pushUMinus(self, strg, loc, toks): + if toks and toks[0] == '-': + self.exprStack.append('unary -') + + def __init__(self, extra_literals=[]): + """ + expop :: '^' + multop :: '*' | '/' + addop :: '+' | '-' + integer :: ['+' | '-'] '0'..'9'+ + atom :: PI | E | real | fn '(' expr ')' | '(' expr ')' + factor :: atom [ expop factor ]* + term :: factor [ multop factor ]* + expr :: term [ addop term ]* + """ + point = pp.Literal(".") + e = pp.CaselessLiteral("E") + fnumber = pp.Combine(pp.Word("+-" + pp.nums, pp.nums) + + pp.Optional(point + pp.Optional(pp.Word(pp.nums))) + + pp.Optional(e + pp.Word("+-" + pp.nums, pp.nums))) + ident = pp.Word(pp.alphas, pp.alphas + pp.nums + "_$") + plus = pp.Literal("+") + minus = pp.Literal("-") + mult = pp.Literal("*") + div = pp.Literal("/") + lpar = pp.Literal("(").suppress() + rpar = pp.Literal(")").suppress() + addop = plus | minus + multop = mult | div + expop = pp.Literal("^") + pi = pp.CaselessLiteral("PI") + + self.extra_literals = extra_literals + pp_extra_literals = functools.reduce(operator.or_, [pp.CaselessLiteral(e) for e in extra_literals], pp.NoMatch()) + + expr = pp.Forward() + atom = ((pp.Optional(pp.oneOf("- +")) + + (pi | e | pp_extra_literals | fnumber | ident + lpar + expr + rpar).setParseAction(self.pushFirst)) + | pp.Optional(pp.oneOf("- +")) + pp.Group(lpar + expr + rpar) + ).setParseAction(self.pushUMinus) + # by defining exponentiation as "atom [ ^ factor ]..." instead of + # "atom [ ^ atom ]...", we get right-to-left exponents, instead of left-to-right + # that is, 2^3^2 = 2^(3^2), not (2^3)^2. + factor = pp.Forward() + factor << atom + pp.ZeroOrMore((expop + factor).setParseAction(self.pushFirst)) + term = factor + pp.ZeroOrMore((multop + factor).setParseAction(self.pushFirst)) + expr << term + pp.ZeroOrMore((addop + term).setParseAction(self.pushFirst)) + # addop_term = ( addop + term ).setParseAction( self.pushFirst ) + # general_term = term + ZeroOrMore( addop_term ) | OneOrMore( addop_term) + # expr << general_term + self.bnf = expr + # map operator symbols to corresponding arithmetic operations + epsilon = 1e-12 + self.opn = {"+": operator.add, + "-": operator.sub, + "*": operator.mul, + "/": operator.truediv, + "^": operator.pow} + self.fn = {"sin": math.sin, + "cos": math.cos, + "tan": math.tan, + "abs": abs, + "trunc": lambda a: int(a), + "round": round, + "sgn": lambda a: abs(a) > epsilon and pp.cmp(a, 0) or 0} + + def evaluateStack(self, s, key_value_map={}): + op = s.pop() + if op == 'unary -': + return -self.evaluateStack(s, key_value_map) + if op in "+-*/^": + op2 = self.evaluateStack(s, key_value_map) + op1 = self.evaluateStack(s, key_value_map) + return self.opn[op](op1, op2) + elif op == "PI": + return math.pi # 3.1415926535 + elif op == "E": + return math.e # 2.718281828 + elif op in self.extra_literals: + return key_value_map[op] + elif op in self.fn: + return self.fn[op](self.evaluateStack(s, key_value_map)) + elif op[0].isalpha(): + return 0 + else: + return float(op) -# TODO: Cache -def eval_formular(s, params={}): - """evaluates the formular. + def eval(self, num_string, key_value_map={}, parseAll=True): + self.exprStack = [] + results = self.bnf.parseString(num_string, parseAll) + val = self.evaluateStack(self.exprStack[:], key_value_map = key_value_map) + return val - parameters are available as p.name, - available math functions: - 'math','acos', 'asin', 'atan', 'atan2', 'ceil', - 'cos', 'cosh', 'degrees', 'e', 'exp', 'fabs', 'floor', - 'fmod', 'frexp', 'hypot', 'ldexp', 'log', 'log10', 'modf', - 'pi', 'pow', 'radians', 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'sum', 'range' - """ +def evaluate_value_expression(expression, params={}): + if expression is None: + return None try: - if s is None: + nsp = NumericStringParser(extra_literals=params.keys()) + return nsp.eval(expression,key_value_map=params) + except: + raise FormularEvaluationException(expression) + + +def render_string(tpl, params): + """Substitute text in <> with corresponding variable value.""" + regex = re.compile('\${(.+)}') + def repl(m): + group = m.group(1) + value = evaluate_value_expression(group, params) + if int(value) == value: + value = int(value) + return str(value) + return regex.sub(repl, tpl) + + +def evaluate_string(inst, params=None): + try: + if inst is None: return None + if params is not None: + formatted = render_string(inst, params) + else: + formatted = inst + + try: + if str(int(formatted)) == str(formatted): + return int(formatted) + except: + pass + + try: + if str(int(float)) == str(float): + return float(formatted) + except: + pass + + return formatted + except: + raise FormularEvaluationException(inst) + + +# The condition JSON-Schema +class Conjunction(jsl.Document): + terms = jsl.ArrayField(jsl.OneOfField([ + jsl.DocumentField("Conjunction", as_ref=True), + jsl.DocumentField("Disjunction", as_ref=True), + jsl.DocumentField("Literal", as_ref=True) + ], required=True), required=True) + type = jsl.StringField(pattern="^conjunction$") + + +class Disjunction(jsl.Document): + terms = jsl.ArrayField(jsl.OneOfField([ + jsl.DocumentField("Conjunction", as_ref=True), + jsl.DocumentField("Disjunction", as_ref=True), + jsl.DocumentField("Literal", as_ref=True) + ], required=True), required=True) + type = jsl.StringField(pattern="^disjunction$") + + +class Literal(jsl.Document): + variable = jsl.StringField(required=True) + key_operator = jsl.StringField(pattern = "^(IN|ILIKE)$", required=False) + key = jsl.ArrayField(jsl.StringField(), required=False) + type = jsl.StringField(pattern="^literal$") + + +class TermDocument(jsl.Document): + term = jsl.OneOfField([ + jsl.DocumentField(Conjunction, as_ref=True), + jsl.DocumentField(Disjunction, as_ref=True), + jsl.DocumentField(Literal, as_ref=True) + ], required=True) + + +def validate_term(condition_term): + return jsonschema.validate(condition_term, TermDocument.get_schema()) + +def _term_eval(term, column_variable, column_key): + + if term["type"].lower() == "conjunction": + return and_(*((_term_eval(t, column_variable, column_key) for t in term["terms"]))) + elif term["type"].lower() == "disjunction": + return or_(*((_term_eval(t, column_variable, column_key) for t in term["terms"]))) + elif term["type"].lower() == "literal": + if "key" in term and term["key"]: + key_operator = term.get("key_operator", "IN") + if key_operator is None or key_operator == "IN": + key_condition = column_key.in_(term["key"]) + elif key_operator=="ILIKE": + key_condition = or_(*(column_key.ilike(pattern) for pattern in term["key"])) + return and_(column_variable==term["variable"], key_condition) else: - p = DictObjectProxy(params) + return column_variable==term["variable"] - # add any needed builtins back in. - safe_dict['p'] = p - result = eval(s, {"__builtins__": None}, safe_dict) - if type(result) == str: - return result % params - else: - return result +def evaluate_condition(inst, column_variable=None, column_key=None): + try: + if isinstance(inst,str): + inst = json.loads(inst) + from gengine.tenant.model import t_values, t_variables + if column_variable is None: + column_variable = t_variables.c.name.label("variable_name") + if column_key is None: + column_key = t_variables.c.name.label("variable_name") + + jsonschema.validate(inst, TermDocument.get_schema()) + return _term_eval(inst["term"], column_variable, column_key) except: - raise FormularEvaluationException(s) \ No newline at end of file + raise FormularEvaluationException(json.dumps(inst)) + + +demo_schema = { + 'term': { + 'variable': 'participate', + 'key_operator' : 'IN', + 'key': ['2', ], + 'type' : 'literal' + } +} + +demo2_schema = { + 'term': { + 'type' : 'disjunction', + 'terms': [ + { + 'type' : 'literal', + 'variable': 'participate', + 'key_operator' : 'ILIKE', + 'key': ['%blah%', ] + }, + { + 'type' : 'literal', + 'variable': 'participate', + 'key_operator' : 'IN', + 'key': ['2', ] + } + ] + } +} diff --git a/gengine/tenant/model.py b/gengine/tenant/model.py index 35ace0a..70302d6 100644 --- a/gengine/tenant/model.py +++ b/gengine/tenant/model.py @@ -28,7 +28,8 @@ from sqlalchemy.sql import bindparam from gengine.metadata import Base, DBSession -from gengine.tenant.formular import eval_formular + +from gengine.tenant.formular import evaluate_condition, evaluate_value_expression, evaluate_string t_users = Table("users", Base.metadata, Column('id', ty.BigInteger, primary_key = True), @@ -65,7 +66,7 @@ Column("achievementcategory_id", ty.Integer, ForeignKey("achievementcategories.id", ondelete="SET NULL"), index=True, nullable=True), Column('name', ty.String(255), nullable = False), #internal use Column('maxlevel',ty.Integer, nullable=False, default=1), - Column('hidden',ty.Boolean, nullable=False, default=False), + Column('hidden',ty.Boolean, nullable=False, default=False), Column('valid_start',ty.Date, nullable=True), Column('valid_end',ty.Date, nullable=True), Column("lat", ty.Float(Precision=64), nullable=True), @@ -90,7 +91,7 @@ Column('maxmin', ty.Enum("max","min", name="goal_maxmin"), nullable=True, default="max"), Column('achievement_id', ty.Integer, ForeignKey("achievements.id", ondelete="CASCADE"), nullable=False), Column('priority', ty.Integer, index=True, default=0), -) +) t_goal_evaluation_cache = Table("goal_evaluation_cache", Base.metadata, Column("goal_id", ty.Integer, ForeignKey("goals.id", ondelete="CASCADE"), primary_key=True, nullable=False), @@ -191,29 +192,29 @@ class User(ABase): """A user participates in the gamification, i.e. can get achievements, rewards, participate in leaderbaord etc.""" - + def __unicode__(self, *args, **kwargs): return "User %s" % (self.id,) - + def __init__(self, *args, **kw): """ create a user object - + Each user has a timezone and a location to support time- and geo-aware gamification. - There is also a friends-relation for leaderboards and a groups-relation. + There is also a friends-relation for leaderboards and a groups-relation. """ ABase.__init__(self, *args, **kw) - + #TODO:Cache @classmethod def get_user(cls,user_id): return DBSession.execute(t_users.select().where(t_users.c.id==user_id)).fetchone() - + @classmethod def get_cache_expiration_time_for_today(cls,user): """return the seconds until the day of the user ends (timezone of the user). - + This is needed as achievements may be limited to a specific time (e.g. only during holidays).""" - + tzobj = pytz.timezone(user["timezone"]) now = datetime.datetime.now(tzobj) today = now.replace(hour=0,minute=0,second=0,microsecond=0) @@ -223,28 +224,28 @@ def get_cache_expiration_time_for_today(cls,user): @classmethod def set_infos(cls,user_id,lat,lng,timezone,country,region,city,friends, groups): """set the user's metadata like friends,location and timezone""" - - + + new_friends_set = set(friends) existing_users_set = {x["id"] for x in DBSession.execute(select([t_users.c.id]).where(t_users.c.id.in_([user_id,]+friends))).fetchall()} existing_friends = {x["to_id"] for x in DBSession.execute(select([t_users_users.c.to_id]).where(t_users_users.c.from_id==user_id)).fetchall()} friends_to_create = (new_friends_set-existing_users_set-{user_id,}) friends_to_append = (new_friends_set-existing_friends) friends_to_delete = (existing_friends-new_friends_set) - + new_groups_set = set(groups) existing_groups_set = {x["id"] for x in DBSession.execute(select([t_groups.c.id]).where(t_groups.c.id.in_(groups))).fetchall()} existing_groups_of_user = {x["group_id"] for x in DBSession.execute(select([t_users_groups.c.group_id]).where(t_users_groups.c.user_id==user_id)).fetchall()} groups_to_create = (new_groups_set-existing_groups_set) groups_to_append = (new_groups_set-existing_groups_of_user) groups_to_delete = (existing_groups_of_user-new_groups_set) - + #add or select user if user_id in existing_users_set: user = DBSession.query(User).filter_by(id=user_id).first() else: user = User() - + user.id = user_id user.lat = lat user.lng = lng @@ -252,40 +253,40 @@ def set_infos(cls,user_id,lat,lng,timezone,country,region,city,friends, groups): user.country = country user.region = region user.city = city - + DBSession.add(user) DBSession.flush() - + #FRIENDS - + #insert missing friends in user table if len(friends_to_create)>0: update_connection().execute(t_users.insert(), [{"id":f} for f in friends_to_create]) - + #delete old friends if len(friends_to_delete)>0: update_connection().execute(t_users_users.delete().where(and_(t_users_users.c.from_id==user_id, t_users_users.c.to_id.in_(friends_to_delete)))) - + #insert missing friends if len(friends_to_append)>0: update_connection().execute(t_users_users.insert(),[{"from_id":user_id,"to_id":f} for f in friends_to_append]) - + #GROUPS - + #insert missing groups in group table if len(groups_to_create)>0: update_connection().execute(t_groups.insert(), [{"id":f} for f in groups_to_create]) - + #delete old groups of user if len(groups_to_delete)>0: update_connection().execute(t_users_groups.delete().where(and_(t_users_groups.c.user_id==user_id, t_users_groups.c.group_id.in_(groups_to_delete)))) - + #insert missing groups of user if len(groups_to_append)>0: update_connection().execute(t_users_groups.insert(),[{"user_id":user_id,"group_id":f} for f in groups_to_append]) - + @classmethod def delete_user(cls,user_id): """delete a user including all dependencies.""" @@ -296,37 +297,37 @@ def delete_user(cls,user_id): update_connection().execute(t_users_groups.delete().where(t_users_groups.c.user_id==user_id)) update_connection().execute(t_values.delete().where(t_values.c.user_id==user_id)) update_connection().execute(t_users.delete().where(t_users.c.id==user_id)) - + class Group(ABase): def __unicode__(self, *args, **kwargs): return "(ID: %s)" % (self.id,) class Variable(ABase): """A Variable is anything that should be meassured in your application and be used in :class:`.Goal`. - + To save database rows, variables may be grouped by time: group needs to be set to "year","month","week","day","timeslot" or "none" (default) """ - + def __unicode__(self, *args, **kwargs): return self.name + " (ID: %s)" % (self.id,) - + @classmethod @cache_general.cache_on_arguments() def get_variable_by_name(cls,name): return DBSession.execute(t_variables.select(t_variables.c.name==name)).fetchone() - + @classmethod def get_datetime_for_tz_and_group(cls,tz,group): """get the datetime of the current row, needed for grouping - + when "timezone" is used as a group name, the values are grouped to the nearest time in (09:00, 12:00, 15:00, 18:00, 21:00) (timezone to use is given as parameter) """ tzobj = pytz.timezone(tz) now = datetime.datetime.now(tzobj) #now = now.replace(tzinfo=pytz.utc) - + t = None if group=="year": t = now.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0 ) @@ -342,14 +343,14 @@ def get_datetime_for_tz_and_group(cls,tz,group): else: #return datetime.datetime.max.replace return datetime.datetime(year=2000,month=1,day=1,hour=0,minute=0,second=0,microsecond=0).replace(tzinfo=pytz.utc) - + return t.astimezone(tzobj) - + @classmethod @cache_general.cache_on_arguments() def map_variables_to_rules(cls): """return a map from variable_ids to {"goal":goal_obj,"achievement":achievement_obj} dictionaries. - + Used to know which goals need to be reevaluated after a value for the variable changes. """ q = select([t_goals,t_variables.c.id.label("variable_id")])\ @@ -359,11 +360,11 @@ def map_variables_to_rules(cls): for row in DBSession.execute(q).fetchall(): if not row["variable_id"] in m: m[row["variable_id"]] = [] - + m[row["variable_id"]].append({"goal":row,"achievement":Achievement.get_achievement(row["achievement_id"])}) - + return m - + @classmethod def invalidate_caches_for_variable_and_user(cls,variable_id,user_id): """ invalidate the relevant caches for this user and all relevant users with concerned leaderboards""" @@ -372,33 +373,33 @@ def invalidate_caches_for_variable_and_user(cls,variable_id,user_id): Goal.clear_goal_caches(user_id, [entry["goal"]["id"] for entry in goalsandachievements]) for entry in goalsandachievements: Achievement.invalidate_evaluate_cache(user_id,entry["achievement"]) - - + + class Value(ABase): """A Value describes the relation of the user to a variable. - + (e.g. it counts the occurences of the "events" which the variable represents) """ - + @classmethod def increase_value(cls, variable_name, user, value, key): """increase the value of the variable for the user. - + In addition to the variable_name there may be an application-specific key which can be used in your :class:`.Goal` definitions """ - + user_id = user["id"] tz = user["timezone"] - + variable = Variable.get_variable_by_name(variable_name) datetime = Variable.get_datetime_for_tz_and_group(tz,variable["group"]) - + condition = and_(t_values.c.datetime==datetime, t_values.c.variable_id==variable["id"], t_values.c.user_id==user_id, t_values.c.key==str(key)) - + current_value = DBSession.execute(select([t_values.c.value,]).where(condition)).scalar() - + if current_value is not None: update_connection().execute(t_values.update(condition, values={"value":current_value+value})) else: @@ -409,67 +410,67 @@ def increase_value(cls, variable_name, user, value, key): "value":value})) Variable.invalidate_caches_for_variable_and_user(variable["id"],user["id"]) - + class AchievementCategory(ABase): """A category for grouping achievement types""" - + @classmethod @cache_general.cache_on_arguments() def get_achievementcategory(cls,achievementcategory_id): return DBSession.execute(t_achievementcategories.select().where(t_achievementcategories.c.id==achievementcategory_id)).fetchone() - + def __unicode__(self, *args, **kwargs): return self.name + " (ID: %s)" % (self.id,) - + class Achievement(ABase): """A collection of goals which has multiple :class:`AchievementProperty` and :class:`Reward`.""" - + def __unicode__(self, *args, **kwargs): return self.name + " (ID: %s)" % (self.id,) - + @classmethod @cache_general.cache_on_arguments() def get_achievement(cls,achievement_id): return DBSession.execute(t_achievements.select().where(t_achievements.c.id==achievement_id)).fetchone() - + @classmethod def get_achievements_by_user_for_today(cls,user): """Returns all achievements that are relevant for the user today. - + This is needed as achievements may be limited to a specific time (e.g. only during holidays) - """ - + """ + def generate_achievements_by_user_for_today(): today = datetime.date.today() by_loc = {x["id"] : x["distance"] for x in cls.get_achievements_by_location(coords(user))} by_date = cls.get_achievements_by_date(today) - + def update(arr,distance): arr["distance"]=distance return arr - + return [update(arr,by_loc[arr["id"]]) for arr in by_date if arr["id"] in by_loc] - + key = str(user["id"]) expiration_time = User.get_cache_expiration_time_for_today(user) - + return cache_achievements_by_user_for_today.get_or_create(key,generate_achievements_by_user_for_today, expiration_time=expiration_time) - + #We need to fetch all achievement data in one of these methods -> by_date is just queried once a date - + @classmethod @cache_general.cache_on_arguments() def get_achievements_by_location(cls,latlng): - """return achievements which are valid in that location.""" + """return achievements which are valid in that location.""" #TODO: invalidate automatically when achievement in user's range is modified distance = calc_distance(latlng, (t_achievements.c.lat, t_achievements.c.lng)).label("distance") - + q = select([t_achievements.c.id, distance])\ .where(or_(and_(t_achievements.c.lat==None,t_achievements.c.lng==None), distance < t_achievements.c.max_distance)) return [dict(x.items()) for x in DBSession.execute(q).fetchall() if len(Goal.get_goals(x['id']))>0] - + @classmethod @cache_general.cache_on_arguments() def get_achievements_by_date(cls,date): @@ -480,69 +481,69 @@ def get_achievements_by_date(cls,date): t_achievements.c.valid_end>=date) )) return [dict(x.items()) for x in DBSession.execute(q).fetchall() if len(Goal.get_goals(x['id']))>0] - + #TODO:CACHE @classmethod def get_relevant_users_by_achievement_and_user(cls,achievement,user_id): - """return all relevant other users for the leaderboard. - + """return all relevant other users for the leaderboard. + depends on the "relevance" attribute of the achivement, can be "friends" or "city" (city is still a todo) """ # this is needed to compute the leaderboards - users=[user_id,] + users=[user_id,] if achievement["relevance"]=="city": #TODO pass elif achievement["relevance"]=="friends": users += [x["to_id"] for x in DBSession.execute(select([t_users_users.c.to_id,], t_users_users.c.from_id==user_id)).fetchall()] return set(users) - + #TODO:CACHE @classmethod def get_relevant_users_by_achievement_and_user_reverse(cls,achievement,user_id): """return all users which have this user as friends and are relevant for this achievement. - + the reversed version is needed to know in whose contact list the user is. when the user's value is updated, all the leaderboards of these users need to be regenerated""" - users=[user_id,] + users=[user_id,] if achievement["relevance"]=="city": #TODO pass elif achievement["relevance"]=="friends": users += [x["from_id"] for x in DBSession.execute(select([t_users_users.c.from_id,], t_users_users.c.to_id==user_id)).fetchall()] return set(users) - + @classmethod def get_level(cls, user_id, achievement_id): """get the current level of the user for this achievement.""" - - def generate(): + + def generate(): q = select([t_achievements_users.c.level, t_achievements_users.c.updated_at], and_(t_achievements_users.c.user_id==user_id, t_achievements_users.c.achievement_id==achievement_id)).order_by(t_achievements_users.c.level.desc()) return [x for x in DBSession.execute(q).fetchall()] - + return cache_achievements_users_levels.get_or_create("%s_%s" % (user_id,achievement_id),generate) - + @classmethod def get_level_int(cls,user_id,achievement_id): """get the current level of the user for this achievement as int (0 if the user does not have this achievement)""" lvls = Achievement.get_level(user_id, achievement_id) - + if not lvls: return 0 else: return lvls[0]["level"] - + @classmethod def basic_output(cls,achievement,goals,include_levels=True, max_level_included=None): """construct the basic basic_output structure for the achievement.""" - + achievementcategory = None if achievement["achievementcategory_id"]!=None: achievementcategory = AchievementCategory.get_achievementcategory(achievement["achievementcategory_id"]) - + out = { "id" : achievement["id"], "internal_name" : achievement["name"], @@ -552,12 +553,12 @@ def basic_output(cls,achievement,goals,include_levels=True, "achievementcategory" : achievementcategory["name"] if achievementcategory!=None else "" #"updated_at" : combine_updated_at([achievement["updated_at"],]), } - + if include_levels: levellimit = achievement["maxlevel"] if max_level_included: - max_level_included = min(max_level_included,levellimit) - + max_level_included = min(max_level_included,levellimit) + out["levels"] = { str(i) : { "level" : i, @@ -566,66 +567,66 @@ def basic_output(cls,achievement,goals,include_levels=True, "id" : r["id"], "reward_id" : r["reward_id"], "name" : r["name"], - "value" : eval_formular(r["value"], {"level":i}), + "value" : evaluate_string(r["value"], {"level":i}), "value_translated" : Translation.trs(r["value_translation_id"], {"level":i}), } for r in Achievement.get_rewards(achievement["id"],i)}, "properties" : {str(r["property_id"]) : { "property_id" : r["property_id"], "name" : r["name"], - "value" : eval_formular(r["value"], {"level":i}), + "value" : evaluate_string(r["value"], {"level":i}), "value_translated" : Translation.trs(r["value_translation_id"], {"level":i}), } for r in Achievement.get_achievement_properties(achievement["id"],i)} } for i in range(0,max_level_included+1)} return out - + @classmethod def evaluate(cls, user, achievement_id): """evaluate the achievement including all its subgoals for the user. - + return the basic_output for the achievement plus information about the new achieved levels """ - + def generate(): user_id = user["id"] achievement = Achievement.get_achievement(achievement_id) user_ids = Achievement.get_relevant_users_by_achievement_and_user(achievement, user_id) - + user_has_level = Achievement.get_level_int(user_id, achievement["id"]) user_wants_level = min((user_has_level or 0)+1, achievement["maxlevel"]) - + goal_evals={} all_goals_achieved = True goals = Goal.get_goals(achievement["id"]) - + for goal in goals: - + goal_eval = Goal.get_goal_eval_cache(goal["id"], user_id) if not goal_eval: Goal.evaluate(goal, user_id, user_wants_level,None) goal_eval = Goal.get_goal_eval_cache(goal["id"], user_id) - + if achievement["relevance"]=="friends" or achievement["relevance"]=="city": goal_eval["leaderboard"] = Goal.get_leaderboard(goal, user_ids) - goal_eval["leaderboard_position"] = filter(lambda x : x["user_id"]==user_id, goal_eval["leaderboard"])[0]["position"] - + goal_eval["leaderboard_position"] = list(filter(lambda x : x["user_id"]==user_id, goal_eval["leaderboard"]))[0]["position"] + goal_evals[goal["id"]]=goal_eval if not goal_eval["achieved"]: all_goals_achieved = False - + output = "" new_level_output = None full_output = True # will be false, if the full basic_output is generated in a recursion step - + if all_goals_achieved and user_has_level < achievement["maxlevel"]: #NEW LEVEL YEAH! - + new_level_output = { "rewards" : { str(r["id"]) : { "id" : r["id"], "reward_id" : r["reward_id"], "name" : r["name"], - "value" : eval_formular(r["value"], {"level":user_wants_level}), + "value" : evaluate_string(r["value"], {"level":user_wants_level}), "value_translated" : Translation.trs(r["value_translation_id"], {"level":user_wants_level}), } for r in Achievement.get_rewards(achievement["id"],user_wants_level) }, @@ -634,36 +635,36 @@ def generate(): "property_id" : r["property_id"], "name" : r["name"], "is_variable" : r["is_variable"], - "value" : eval_formular(r["value"], {"level":user_wants_level}), + "value" : evaluate_string(r["value"], {"level":user_wants_level}), "value_translated" : Translation.trs(r["value_translation_id"], {"level":user_wants_level}) } for r in Achievement.get_achievement_properties(achievement["id"],user_wants_level) }, "level" : user_wants_level } - + for prop in new_level_output["properties"].values(): if prop["is_variable"]: Value.increase_value(prop["name"], user, prop["value"], achievement_id) - + update_connection().execute(t_achievements_users.insert().values({ "user_id" : user_id, "achievement_id" : achievement["id"], "level" : user_wants_level })) - + #invalidate getter cache_achievements_users_levels.delete("%s_%s" % (user_id,achievement_id)) - + user_has_level = user_wants_level user_wants_level = user_wants_level+1 - + Goal.clear_goal_caches(user_id, [g["goal_id"] for g in goal_evals.values()]) #the level has been updated, we need to do recursion now... #but only if there are more levels... if user_has_level < achievement["maxlevel"]: output = generate() full_output = False - + if full_output: #is executed, if this is the last recursion step output = Achievement.basic_output(achievement,goals,True,max_level_included=user_has_level+1) output.update({ @@ -676,28 +677,28 @@ def generate(): "goals":goal_evals, #"updated_at":combine_updated_at([achievement["updated_at"],] + [g["updated_at"] for g in goal_evals]) }) - + if new_level_output is not None: #if we reached a new level in this recursion step, add the previous levels rewards and properties output["new_levels"][str(user_has_level)]=new_level_output - + return output - + return cache_achievement_eval.get_or_create("%s_%s" % (user["id"],achievement_id),generate) - + @classmethod def invalidate_evaluate_cache(cls,user_id,achievement): """invalidate the evaluation cache for all goals of this achievement for the user.""" - + #We neeed to invalidate for all relevant users because of the leaderboards for uid in Achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user_id): cache_achievement_eval.delete("%s_%s" % (uid,achievement["id"])) invalidate("/progress/"+str(uid)) - + @classmethod @cache_general.cache_on_arguments() def get_rewards(cls,achievement_id,level): """return the new rewards which are given for the achievement level.""" - + this_level = DBSession.execute(select([t_rewards.c.id.label("reward_id"), t_achievements_rewards.c.id, t_rewards.c.name, @@ -710,7 +711,7 @@ def get_rewards(cls,achievement_id,level): t_achievements_rewards.c.achievement_id==achievement_id))\ .order_by(t_achievements_rewards.c.from_level))\ .fetchall() - + prev_level = DBSession.execute(select([t_rewards.c.id.label("reward_id"), t_achievements_rewards.c.id, t_achievements_rewards.c.value, @@ -722,19 +723,19 @@ def get_rewards(cls,achievement_id,level): .order_by(t_achievements_rewards.c.from_level))\ .fetchall() #now compute the diff :-/ - build_hash = lambda x,l : hashlib.md5(str(x["id"])+str(eval_formular(x["value"], {"level":l}))+str(Translation.trs(x["value_translation_id"], {"level":l}))).hexdigest() - + build_hash = lambda x,l : hashlib.md5((str(x["id"])+str(evaluate_string(x["value"], {"level":l}))+str(Translation.trs(x["value_translation_id"], {"level":l}))).encode("UTF-8")).hexdigest() + prev_hashes = {build_hash(x,level-1) for x in prev_level} - this_hashes = {build_hash(x,level) for x in this_level} - + #this_hashes = {build_hash(x,level) for x in this_level} + retlist = [x for x in this_level if not build_hash(x,level) in prev_hashes] return retlist - + @classmethod @cache_general.cache_on_arguments() def get_achievement_properties(cls,achievement_id,level): """return all properties which are associated to the achievement level.""" - + return DBSession.execute(select([t_achievementproperties.c.id.label("property_id"), t_achievementproperties.c.name, t_achievementproperties.c.is_variable, @@ -747,16 +748,16 @@ def get_achievement_properties(cls,achievement_id,level): t_achievements_achievementproperties.c.achievement_id==achievement_id))\ .order_by(t_achievements_achievementproperties.c.from_level))\ .fetchall() - - + + class AchievementProperty(ABase): """A AchievementProperty describes the :class:`Achievement`s of our system. - + Examples: name, image, description, xp - + Additionally Properties can be used as variables. - This is useful to model goals like "reach 1000xp" - + This is useful to model goals like "reach 1000xp" + """ def __unicode__(self, *args, **kwargs): return self.name + " (ID: %s)" % (self.id,) @@ -764,15 +765,15 @@ def __unicode__(self, *args, **kwargs): class AchievementAchievementProperty(ABase): """A poperty value for an :class:`Achievement`""" pass - + class GoalProperty(ABase): """A goalproperty describes the :class:`Goal`s of our system. - + Examples: name, image, description, xp - + Additionally Properties can be used as variables. - This is useful to model goals like "reach 1000xp" - + This is useful to model goals like "reach 1000xp" + """ def __unicode__(self, *args, **kwargs): return self.name + " (ID: %s)" % (self.id,) @@ -780,10 +781,10 @@ def __unicode__(self, *args, **kwargs): class GoalGoalProperty(ABase): """A goalpoperty value for a :class:`Goal`""" pass - + class Reward(ABase): """Rewards are given when reaching :class:`Achievement`s. - + Examples: badge, item """ def __unicode__(self, *args, **kwargs): @@ -791,7 +792,7 @@ def __unicode__(self, *args, **kwargs): class AchievementReward(ABase): """A Reward value for an :class:`Achievement` """ - + @classmethod def get_achievement_reward(cls, achievement_reward_id): return DBSession.execute(t_achievements_rewards.select(t_achievements_rewards.c.id==achievement_reward_id)).fetchone() @@ -799,87 +800,87 @@ def get_achievement_reward(cls, achievement_reward_id): class AchievementUser(ABase): """Relation between users and achievements, contains level and updated_at date""" pass - + class GoalEvaluationCache(ABase): """Cache for the evaluation of goals for users""" pass class Goal(ABase): """A Goal defines a rule on variables that needs to be reached to get achievements""" - + def __unicode__(self, *args, **kwargs): if self.name_translation!=None: name = Translation.trs(self.name_translation.id, {"level":1, "goal":'0'})[_fallback_language] return str(name) + " (ID: %s)" % (self.id,) else: return self.name + " (ID: %s)" % (self.id,) - + @classmethod @cache_general.cache_on_arguments() def get_goals(cls,achievement_id): return DBSession.execute(t_goals.select(t_goals.c.achievement_id==achievement_id)).fetchall() - + @classmethod def compute_progress(cls,goal,user_id): """computes the progress of the goal for the given user_id - + goal attributes: - goal: the value that is used for comparison - operator: "geq" or "leq"; used for comparison - condition: the rule as python code - group_by_dateformat: passed as a parameter to to_char ( http://www.postgresql.org/docs/9.3/static/functions-formatting.html ) e.g. you can select and group by the weekday by using "ID" for ISO 8601 day of the week (1-7) which can afterwards be used in the condition - + - group_by_key: group by the key of the values table - timespan: number of days which are considered (uses utc, i.e. days*24hours) - maxmin: "max" or "min" - select min or max value after grouping - evaluation: "daily", "weekly", "monthly", "yearly" evaluation (users timezone) - + """ - + def generate_statement_cache(): - condition = eval_formular(goal["condition"],{"var" : t_variables.c.name.label("variable_name"), - "key" : t_values.c.key}) + condition = evaluate_condition(goal["condition"], column_variable = t_variables.c.name.label("variable_name"), + column_key = t_values.c.key) group_by_dateformat = goal["group_by_dateformat"] group_by_key = goal["group_by_key"] timespan = goal["timespan"] maxmin = goal["maxmin"] evaluation_type = goal["evaluation"] - + #prepare select_cols=[func.sum(t_values.c.value).label("value"), t_values.c.user_id] - + j = t_values.join(t_variables) - + if evaluation_type in ("daily","weekly","monthly","yearly"): # We need to access the user's timezone later j = j.join(t_users) - + datetime_col=None if group_by_dateformat: # here we need to convert to users' time zone, as we might need to group by e.g. USER's weekday datetime_col = func.to_char(text("values.datetime AT TIME ZONE users.timezone"),group_by_dateformat).label("datetime") select_cols.append(datetime_col) - + if group_by_key: select_cols.append(t_values.c.key) - + #build query q = select(select_cols, from_obj=j)\ .where(t_values.c.user_id==bindparam("user_id"))\ .group_by(t_values.c.user_id) - + if condition is not None: q = q.where(condition) - + if timespan: #here we can use the utc time q = q.where(t_values.c.datetime>=datetime.datetime.utcnow()-datetime.timedelta(days=timespan)) - + if evaluation_type!="immediately": - + if evaluation_type=="daily": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("day","users.timezone"))) elif evaluation_type=="weekly": @@ -888,74 +889,74 @@ def generate_statement_cache(): q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("month","users.timezone"))) elif evaluation_type=="yearly": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("year","users.timezone"))) - + if datetime_col or group_by_key: if datetime_col: q = q.group_by(datetime_col) - + if group_by_key: q = q.group_by(t_values.c.key) - + query_with_groups = q.alias() - + select_cols2 = [query_with_groups.c.user_id] - + if maxmin=="min": select_cols2.append(func.min(query_with_groups.c.value).label("value")) else: select_cols2.append(func.max(query_with_groups.c.value).label("value")) - + combined_user_query = select(select_cols2,from_obj=query_with_groups)\ .group_by(query_with_groups.c.user_id) - + return combined_user_query else: return q - + q = cache_goal_statements.get_or_create(str(goal["id"]),generate_statement_cache) - return DBSession.bind.execute(q, user_id=user_id) + return DBSession.execute(q, {'user_id' : user_id}) @classmethod def evaluate(cls, goal, user_id, level, goal_eval_cache_before=False): """evaluate the goal for the user_ids and the level""" - + operator = goal["operator"] - + users_progress = Goal.compute_progress(goal,user_id) - + goal_evaluation = {e["user_id"] : e["value"] for e in users_progress} - + goal_achieved = False if goal_eval_cache_before is False: goal_eval_cache_before = cls.get_goal_eval_cache(goal["id"], user_id) - + new = goal_evaluation.get(user_id,0.0) - + if goal_eval_cache_before is None or goal_eval_cache_before.get("value",0.0)!=goal_evaluation.get(user_id,0.0): - + #Level is the next level, or the current level if I'm alread at max params = { "level" : level } - goal_goal = eval_formular(goal["goal"], params) + goal_goal = evaluate_value_expression(goal["goal"], params) if goal_goal is not None and operator=="geq" and new>=goal_goal: goal_achieved = True new = min(new,goal_goal) - + elif goal_goal is not None and operator=="leq" and new<=goal_goal: goal_achieved = True new = max(new,goal_goal) - + return Goal.set_goal_eval_cache(goal=goal, user_id=user_id, value=new, achieved = goal_achieved) else: return Goal.get_goal_eval_cache(goal["id"], user_id) - + @classmethod def get_goal_eval_cache(cls,goal_id,user_id): """lookup and return cache entry, else return None""" @@ -964,14 +965,15 @@ def get_goal_eval_cache(cls,goal_id,user_id): return v else: return None - + @classmethod def set_goal_eval_cache(cls,goal,user_id,value,achieved): """set cache entry after evaluation""" - - cache = t_goal_evaluation_cache.select().where(and_(t_goal_evaluation_cache.c.goal_id==goal["id"], - t_goal_evaluation_cache.c.user_id==user_id)).execute().fetchone() - + + cache_query = t_goal_evaluation_cache.select().where(and_(t_goal_evaluation_cache.c.goal_id==goal["id"], + t_goal_evaluation_cache.c.user_id==user_id)) + cache = DBSession.execute(cache_query).fetchone() + if not cache: q = t_goal_evaluation_cache.insert()\ .values({"user_id":user_id, @@ -987,7 +989,7 @@ def set_goal_eval_cache(cls,goal,user_id,value,achieved): .values({"value" : value, "achieved" : achieved}) update_connection().execute(q) - + data = { "id" : goal["id"], "value" : value, @@ -997,23 +999,23 @@ def set_goal_eval_cache(cls,goal,user_id,value,achieved): "achievement_id" : goal["achievement_id"], "priority" : goal["priority"] } - + achievement_id = goal["achievement_id"] achievement = Achievement.get_achievement(achievement_id) - + level = min((Achievement.get_level_int(user_id, achievement["id"]) or 0)+1,achievement["maxlevel"]) - + goal_output = Goal.basic_goal_output(data,level) - + goal_output.update({ "achieved" : achieved, "value" : value, }) - + cache_goal_evaluation.set("%s_%s" % (goal["id"],user_id),goal_output) - + return goal_output - + @classmethod def clear_goal_caches(cls, user_id, goal_ids): """clear the evaluation cache for the user and gaols""" @@ -1031,36 +1033,36 @@ def get_leaderboard(cls, goal, user_ids): .order_by(t_goal_evaluation_cache.c.value.desc(), t_goal_evaluation_cache.c.user_id.desc()) items = DBSession.execute(q).fetchall() - + missing_users = set(user_ids)-set([x["user_id"] for x in items]) if len(missing_users)>0: #the goal has not been evaluated for some users... achievement = Achievement.get_achievement(goal["achievement_id"]) - + for user_id in missing_users: user = User.get_user(user_id) - + user_has_level = Achievement.get_level_int(user_id, achievement["id"]) user_wants_level = min((user_has_level or 0)+1, achievement["maxlevel"]) - + goal_eval = Goal.evaluate(goal, user_id, user_wants_level) - + #rerun the query items = DBSession.execute(q).fetchall() - + positions = [{ "user_id" : items[i]["user_id"], "value" : items[i]["value"], "position" : i} for i in range(0,len(items))] - + return positions - + @classmethod @cache_general.cache_on_arguments() def get_goal_properties(cls,goal_id,level): """return all properties which are associated to the achievement level.""" - + #NOT CACHED, as full-basic_output is cached (see Goal.basic_output) - + return DBSession.execute(select([t_goalproperties.c.id.label("property_id"), t_goalproperties.c.name, t_goalproperties.c.is_variable, @@ -1073,16 +1075,16 @@ def get_goal_properties(cls,goal_id,level): t_goals_goalproperties.c.goal_id==goal_id))\ .order_by(t_goals_goalproperties.c.from_level))\ .fetchall() - + @classmethod @cache_general.cache_on_arguments() def basic_goal_output(cls,goal,level): - goal_goal = eval_formular(goal["goal"], {"level":level}) + goal_goal = evaluate_value_expression(goal["goal"], {"level":level}) properties = { str(r["property_id"]) : { "property_id" : r["property_id"], "name" : r["name"], - "value" : eval_formular(r["value"], {"level":level}), + "value" : evaluate_string(r["value"], {"level":level}), "value_translated" : Translation.trs(r["value_translation_id"], {"level":level}), } for r in Goal.get_goal_properties(goal["id"],level) } @@ -1094,7 +1096,7 @@ def basic_goal_output(cls,goal,level): "properties" : properties, #"updated_at" : goal["updated_at"] } - + class Language(ABase): def __unicode__(self, *args, **kwargs): @@ -1108,16 +1110,16 @@ def __unicode__(self, *args, **kwargs): class Translation(ABase): def __unicode__(self, *args, **kwargs): return "%s" % (self.text,) - + @classmethod @cache_translations.cache_on_arguments() def trs(cls,translation_id,params={}): """returns a map of translations for the translation_id for ALL languages""" - + if translation_id is None: - return None + return None try: - ret = {str(x["name"]) : eval_formular(x["text"],params) for x in cls.get_translation_variable(translation_id)} + ret = {str(x["name"]) : evaluate_string(x["text"],params) for x in cls.get_translation_variable(translation_id)} except: ret = {str(x["name"]) : x["text"] for x in cls.get_translation_variable(translation_id)} diff --git a/gengine/tenant/views.py b/gengine/tenant/views.py index 9b1f05c..a797048 100644 --- a/gengine/tenant/views.py +++ b/gengine/tenant/views.py @@ -203,7 +203,7 @@ def increase_multi_values(request): Value.increase_value(variable_name, user, value, key) output = copy.deepcopy(_get_progress(user, force_generation=True)[1]) # 1 is the map - + for aid in list(output["achievements"].keys()): if len(output["achievements"][aid]["new_levels"])>0: if "levels" in output["achievements"][aid]: From 1c66e1258a7354218b98997caecdb8e954470abc Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 15 Jun 2016 09:17:06 +0200 Subject: [PATCH 014/176] - monkey patch flask_admin.menu.MenuView to disable url-caching (otherwise we cannot use the same app-instance for multiple tenants) - create a thread-local context that holds the current tenant in each request. this is then used in the cache-key generation function --- gengine/__init__.py | 10 ++++++++++ gengine/base/cache.py | 16 +++++++++++----- gengine/base/context.py | 13 +++++++++++++ gengine/base/monkeypatch_flaskadmin.py | 6 ++++++ gengine/base/util.py | 12 ++++++++++++ gengine/resources.py | 6 ++++-- gengine/tenant/admin.py | 1 + 7 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 gengine/base/context.py create mode 100644 gengine/base/monkeypatch_flaskadmin.py create mode 100644 gengine/base/util.py diff --git a/gengine/__init__.py b/gengine/__init__.py index b9c0bc5..2ea98cc 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +from pyramid.events import NewRequest + +from gengine.base.context import reset_context __version__ = '0.2.0' @@ -24,8 +27,15 @@ def main(global_config, **settings): init_declarative_base() init_db(engine) + from gengine.base.monkeypatch_flaskadmin import do_monkeypatch + do_monkeypatch() + from gengine.resources import root_factory config = Configurator(settings=settings, root_factory=root_factory) + + def reset_context_on_new_request(event): + reset_context() + config.add_subscriber(reset_context_on_new_request,NewRequest) config.include('pyramid_dogpile_cache') durl = os.environ.get("DATABASE_URL") #heroku diff --git a/gengine/base/cache.py b/gengine/base/cache.py index 7b2925c..5aba16a 100644 --- a/gengine/base/cache.py +++ b/gengine/base/cache.py @@ -1,9 +1,9 @@ import warnings from dogpile.cache import make_region from pyramid_dogpile_cache import get_region +from gengine.base.context import get_context - -def my_key_mangler(prefix): +def my_key_mangler(prefix, is_tenant_cache): def s(o): if type(o) == dict: return "_".join(["%s=%s" % (str(k), str(v)) for k, v in o.items()]) @@ -15,12 +15,18 @@ def s(o): return str(o) def generate_key(key): - return prefix + s(key).replace(" ", "") + ret = "" + if is_tenant_cache: + ret += get_context().tenant.id + "_" + else: + ret += "global_" + ret += prefix + s(key).replace(" ", "") + return ret return generate_key -def create_cache(name): +def create_cache(name, is_tenant_cache=True): ch = None try: @@ -30,7 +36,7 @@ def create_cache(name): ch = make_region().configure('dogpile.cache.memory') warnings.warn("Warning: cache objects are in memory, are you creating docs?") - ch.key_mangler = my_key_mangler(name) + ch.key_mangler = my_key_mangler(name, is_tenant_cache) return ch diff --git a/gengine/base/context.py b/gengine/base/context.py new file mode 100644 index 0000000..4ed1047 --- /dev/null +++ b/gengine/base/context.py @@ -0,0 +1,13 @@ +import threading + +from gengine.base.util import DictObjectProxy + +_local = threading.local() + +def get_context(): + if not hasattr(_local, "context"): + _local.context = DictObjectProxy() + return _local.context + +def reset_context(): + _local.context = DictObjectProxy() \ No newline at end of file diff --git a/gengine/base/monkeypatch_flaskadmin.py b/gengine/base/monkeypatch_flaskadmin.py new file mode 100644 index 0000000..f50a147 --- /dev/null +++ b/gengine/base/monkeypatch_flaskadmin.py @@ -0,0 +1,6 @@ +def do_monkeypatch(): + def get_url(self): + return self._view.get_url('%s.%s' % (self._view.endpoint, self._view._default_view)) + + import flask_admin.menu + flask_admin.menu.MenuView.get_url = get_url \ No newline at end of file diff --git a/gengine/base/util.py b/gengine/base/util.py new file mode 100644 index 0000000..9710609 --- /dev/null +++ b/gengine/base/util.py @@ -0,0 +1,12 @@ +class DictObjectProxy: + + def __init__(self, obj={}): + super().__setattr__("obj",obj) + + def __getattr__(self, name): + if not name in super().__getattribute__("obj"): + raise AttributeError + return super().__getattribute__("obj")[name] + + def __setattr__(self, key, value): + super().__getattribute__("obj")[key] = value diff --git a/gengine/resources.py b/gengine/resources.py index b2d7579..a12beeb 100644 --- a/gengine/resources.py +++ b/gengine/resources.py @@ -2,13 +2,14 @@ from gengine.metadata import DBSession from gengine.olymp.model import t_tenants - +from gengine.base.context import get_context def root_factory(request): return RootResource() class RootResource(): def __getitem__(self, item): + get_context().tenant = None DBSession.execute("SET search_path TO olymp") if item=="t": return TenantCollectionResource(self, "t") @@ -23,7 +24,8 @@ def __getitem__(self, item): q = select([t_tenants.c.id,], from_obj = t_tenants).where(t_tenants.c.id==item) tenant = DBSession.execute(q).fetchone() if tenant: - DBSession.execute("SET search_path TO t_%s" % tenant["id"]) + get_context().tenant = tenant + DBSession.execute("SET search_path TO t_%s" % (tenant["id"],)) return TenantResource(self, tenant) raise KeyError() diff --git a/gengine/tenant/admin.py b/gengine/tenant/admin.py index 1f46d2a..73871d5 100644 --- a/gengine/tenant/admin.py +++ b/gengine/tenant/admin.py @@ -55,6 +55,7 @@ def init_admin(urlprefix="",secret="fKY7kJ2xSrbPC5yieEjV",override_admin=None,ov tenantadminapp.jinja_loader, jinja2.FileSystemLoader(resole_uri("gengine:tenant/templates")), ]) + tenantadminapp.jinja_loader = my_loader tenantadminapp.add_url_rule('/static_gengine/', From c46b7f236c8dd7d25e8ae172325df102bcf3d850 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 24 Jun 2016 16:57:22 +0200 Subject: [PATCH 015/176] include schema conventions --- gengine/metadata.py | 11 ++++++++++- setup.py | 7 ++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/gengine/metadata.py b/gengine/metadata.py index 249de08..486e7f7 100644 --- a/gengine/metadata.py +++ b/gengine/metadata.py @@ -1,6 +1,7 @@ from sqlalchemy.orm.session import Session, sessionmaker import transaction from sqlalchemy.orm.scoping import scoped_session +from sqlalchemy.sql.schema import MetaData from zope.sqlalchemy.datamanager import ZopeTransactionExtension from sqlalchemy.ext.declarative.api import declarative_base @@ -28,7 +29,15 @@ def init_declarative_base(override_base=None): if override_base: Base=override_base else: - Base = declarative_base() + convention = { + "ix": 'ix_%(column_0_label)s', + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s" + } + metadata = MetaData(naming_convention=convention) + Base = declarative_base(metadata = metadata) def init_db(engine): DBSession.configure(bind=engine) diff --git a/setup.py b/setup.py index 2796607..20b62ca 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,10 @@ 'pymemcache', 'mock', 'alembic', - 'raven' + 'raven', + 'jsl', + 'jsonschema', + 'pyparsing' ] version = '' @@ -68,5 +71,7 @@ initialize_gengine_db = gengine.maintenance.scripts.initializedb:main gengine_quickstart = gengine.maintenance.scripts.quickstart:main generate_gengine_erd = gengine.maintenance.scripts.generate_erd:main + [redgalaxy.plugins] + gengine = gengine:redgalaxy """, ) From d37a8cda6a4faf8742696cb08c93cdc6d7d6fb77 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 24 Jun 2016 17:20:43 +0200 Subject: [PATCH 016/176] removing multi-tenancy again (we think an in-app separation is not secure enough; so seperate processes / databases will be used; app-server maintained by uwsgi emperor mode) --- gengine/__init__.py | 18 ++-- gengine/{olymp => app}/__init__.py | 0 gengine/{tenant => app}/admin.py | 32 ++++---- gengine/{olymp => app}/alembic/README | 0 gengine/{olymp => app}/alembic/env.py | 0 gengine/{olymp => app}/alembic/script.py.mako | 0 gengine/{olymp => app}/alembic/versions/DUMMY | 0 gengine/{tenant => app}/formular.py | 2 +- gengine/{tenant => app}/model.py | 2 +- gengine/app/route.py | 9 ++ gengine/{tenant => app}/static/admin.js | 0 .../{tenant => app}/static/admin_layout.css | 0 gengine/{tenant => app}/static/api.js | 0 .../templates/admin/index.html | 0 .../templates/admin/layout.html | 0 .../templates/admin_layout.html | 0 .../templates/admin_maintenance.html | 0 gengine/{olymp => app}/templates/error.html | 0 gengine/{tenant => app}/views.py | 9 +- gengine/base/cache.py | 12 +-- gengine/base/model.py | 12 ++- gengine/maintenance/scripts/initializedb.py | 27 ++---- gengine/olymp/admin.py | 82 ------------------- gengine/olymp/model.py | 46 ----------- gengine/olymp/route.py | 3 - gengine/olymp/templates/admin/index.html | 37 --------- gengine/olymp/templates/admin_layout.html | 12 --- gengine/olymp/views.py | 14 ---- gengine/resources.py | 36 -------- gengine/tenant/__init__.py | 0 gengine/tenant/alembic/README | 1 - gengine/tenant/alembic/env.py | 66 --------------- gengine/tenant/alembic/script.py.mako | 24 ------ gengine/tenant/alembic/versions/DUMMY | 1 - gengine/tenant/route.py | 9 -- gengine/tenant/templates/admin/layout.html | 81 ------------------ gengine/tenant/templates/error.html | 20 ----- 37 files changed, 57 insertions(+), 498 deletions(-) rename gengine/{olymp => app}/__init__.py (100%) rename gengine/{tenant => app}/admin.py (89%) rename gengine/{olymp => app}/alembic/README (100%) rename gengine/{olymp => app}/alembic/env.py (100%) rename gengine/{olymp => app}/alembic/script.py.mako (100%) rename gengine/{olymp => app}/alembic/versions/DUMMY (100%) rename gengine/{tenant => app}/formular.py (99%) rename gengine/{tenant => app}/model.py (99%) create mode 100644 gengine/app/route.py rename gengine/{tenant => app}/static/admin.js (100%) rename gengine/{tenant => app}/static/admin_layout.css (100%) rename gengine/{tenant => app}/static/api.js (100%) rename gengine/{tenant => app}/templates/admin/index.html (100%) rename gengine/{olymp => app}/templates/admin/layout.html (100%) rename gengine/{tenant => app}/templates/admin_layout.html (100%) rename gengine/{tenant => app}/templates/admin_maintenance.html (100%) rename gengine/{olymp => app}/templates/error.html (100%) rename gengine/{tenant => app}/views.py (96%) delete mode 100644 gengine/olymp/admin.py delete mode 100644 gengine/olymp/model.py delete mode 100644 gengine/olymp/route.py delete mode 100644 gengine/olymp/templates/admin/index.html delete mode 100644 gengine/olymp/templates/admin_layout.html delete mode 100644 gengine/olymp/views.py delete mode 100644 gengine/resources.py delete mode 100644 gengine/tenant/__init__.py delete mode 100644 gengine/tenant/alembic/README delete mode 100644 gengine/tenant/alembic/env.py delete mode 100644 gengine/tenant/alembic/script.py.mako delete mode 100644 gengine/tenant/alembic/versions/DUMMY delete mode 100644 gengine/tenant/route.py delete mode 100644 gengine/tenant/templates/admin/layout.html delete mode 100644 gengine/tenant/templates/error.html diff --git a/gengine/__init__.py b/gengine/__init__.py index 2ea98cc..0dc1a69 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -30,8 +30,7 @@ def main(global_config, **settings): from gengine.base.monkeypatch_flaskadmin import do_monkeypatch do_monkeypatch() - from gengine.resources import root_factory - config = Configurator(settings=settings, root_factory=root_factory) + config = Configurator(settings=settings) def reset_context_on_new_request(event): reset_context() @@ -59,22 +58,15 @@ def reset_context_on_new_request(event): urlcache_active = asbool(os.environ.get("URLCACHE_ACTIVE", settings.get("urlcache_active",True))) #routes - from gengine.tenant.route import config_routes as config_tenant_routes - from gengine.olymp.route import config_routes as config_olymp_routes + from gengine.app.route import config_routes as config_tenant_routes config.include(config_tenant_routes, route_prefix=urlprefix) - config.include(config_olymp_routes, route_prefix=urlprefix) - config.add_route('admin_tenant', '/t/{tenant}/*subpath', traverse="/t/{tenant}") #prefix is set in flaskadmin.py - config.add_route('admin_olymp', '/olymp/*subpath') # prefix is set in flaskadmin.py + config.add_route('admin_tenant', '/*subpath') #prefix is set in flaskadmin.py - from gengine.tenant.admin import init_admin as init_tenantadmin + from gengine.app.admin import init_admin as init_tenantadmin init_tenantadmin(urlprefix=urlprefix, - secret=settings.get("flaskadmin_secret","fKY7kJ2xSrbPC5yieEjV")) - - from gengine.olymp.admin import init_admin as init_olympadmin - init_olympadmin(urlprefix=urlprefix, - secret=settings.get("flaskadmin_secret", "fKY7kJ2xSrbPC5yieEjV")) + secret=settings.get("flaskadmin_secret","fKY7kJ2xSrbPC5yieEjV")) from gengine.base.cache import setup_urlcache setup_urlcache(prefix=urlprefix, diff --git a/gengine/olymp/__init__.py b/gengine/app/__init__.py similarity index 100% rename from gengine/olymp/__init__.py rename to gengine/app/__init__.py diff --git a/gengine/tenant/admin.py b/gengine/app/admin.py similarity index 89% rename from gengine/tenant/admin.py rename to gengine/app/admin.py index 73871d5..aaf2b67 100644 --- a/gengine/tenant/admin.py +++ b/gengine/app/admin.py @@ -13,10 +13,10 @@ from wtforms import BooleanField from wtforms.form import Form -from gengine.tenant.model import DBSession, Variable, Goal, AchievementCategory, Achievement, AchievementProperty, GoalProperty, AchievementAchievementProperty, AchievementReward,\ +from gengine.app.model import DBSession, Variable, Goal, AchievementCategory, Achievement, AchievementProperty, GoalProperty, AchievementAchievementProperty, AchievementReward,\ GoalGoalProperty, Reward, User, GoalEvaluationCache, Value, AchievementUser, TranslationVariable, Language, Translation -tenantadminapp=None +adminapp=None admin=None @@ -38,36 +38,36 @@ def send_static_file(filename): return send_static_file def init_admin(urlprefix="",secret="fKY7kJ2xSrbPC5yieEjV",override_admin=None,override_flaskadminapp=None): - global tenantadminapp, admin + global adminapp, admin if not override_flaskadminapp: - tenantadminapp = Flask(__name__) - tenantadminapp.debug=True - tenantadminapp.secret_key = secret - tenantadminapp.config.update(dict( + adminapp = Flask(__name__) + adminapp.debug=True + adminapp.secret_key = secret + adminapp.config.update(dict( PREFERRED_URL_SCHEME = 'https' )) else: - tenantadminapp = override_flaskadminapp + adminapp = override_flaskadminapp # lets add our template directory my_loader = jinja2.ChoiceLoader([ - tenantadminapp.jinja_loader, - jinja2.FileSystemLoader(resole_uri("gengine:tenant/templates")), + adminapp.jinja_loader, + jinja2.FileSystemLoader(resole_uri("gengine:app/templates")), ]) - tenantadminapp.jinja_loader = my_loader + adminapp.jinja_loader = my_loader - tenantadminapp.add_url_rule('/static_gengine/', - endpoint='static_gengine', - view_func=get_static_view('gengine:tenant/static', tenantadminapp)) + adminapp.add_url_rule('/static_gengine/', + endpoint='static_gengine', + view_func=get_static_view('gengine:app/static', adminapp)) - @tenantadminapp.context_processor + @adminapp.context_processor def inject_version(): return { "gamification_engine_version" : pkg_resources.get_distribution("gamification-engine").version } if not override_admin: - admin = Admin(tenantadminapp, + admin = Admin(adminapp, name="Gamification Engine - Admin Control Panel", base_template='admin_layout.html', url=urlprefix+"/admin" diff --git a/gengine/olymp/alembic/README b/gengine/app/alembic/README similarity index 100% rename from gengine/olymp/alembic/README rename to gengine/app/alembic/README diff --git a/gengine/olymp/alembic/env.py b/gengine/app/alembic/env.py similarity index 100% rename from gengine/olymp/alembic/env.py rename to gengine/app/alembic/env.py diff --git a/gengine/olymp/alembic/script.py.mako b/gengine/app/alembic/script.py.mako similarity index 100% rename from gengine/olymp/alembic/script.py.mako rename to gengine/app/alembic/script.py.mako diff --git a/gengine/olymp/alembic/versions/DUMMY b/gengine/app/alembic/versions/DUMMY similarity index 100% rename from gengine/olymp/alembic/versions/DUMMY rename to gengine/app/alembic/versions/DUMMY diff --git a/gengine/tenant/formular.py b/gengine/app/formular.py similarity index 99% rename from gengine/tenant/formular.py rename to gengine/app/formular.py index 643a64e..b51fe9d 100644 --- a/gengine/tenant/formular.py +++ b/gengine/app/formular.py @@ -228,7 +228,7 @@ def evaluate_condition(inst, column_variable=None, column_key=None): try: if isinstance(inst,str): inst = json.loads(inst) - from gengine.tenant.model import t_values, t_variables + from gengine.app.model import t_values, t_variables if column_variable is None: column_variable = t_variables.c.name.label("variable_name") if column_key is None: diff --git a/gengine/tenant/model.py b/gengine/app/model.py similarity index 99% rename from gengine/tenant/model.py rename to gengine/app/model.py index 70302d6..3b2350a 100644 --- a/gengine/tenant/model.py +++ b/gengine/app/model.py @@ -29,7 +29,7 @@ from gengine.metadata import Base, DBSession -from gengine.tenant.formular import evaluate_condition, evaluate_value_expression, evaluate_string +from gengine.app.formular import evaluate_condition, evaluate_value_expression, evaluate_string t_users = Table("users", Base.metadata, Column('id', ty.BigInteger, primary_key = True), diff --git a/gengine/app/route.py b/gengine/app/route.py new file mode 100644 index 0000000..102f99e --- /dev/null +++ b/gengine/app/route.py @@ -0,0 +1,9 @@ + +def config_routes(config): + config.add_route('get_progress', '/progress/{user_id}') + config.add_route('increase_value', '/increase_value/{variable_name}/{user_id}') + config.add_route('increase_value_with_key', '/increase_value/{variable_name}/{user_id}/{key}') + config.add_route('increase_multi_values', '/increase_multi_values') + config.add_route('add_or_update_user', '/add_or_update_user/{user_id}') + config.add_route('delete_user', '/delete_user/{user_id}') + config.add_route('get_achievement_level', '/achievement/{achievement_id}/level/{level}') \ No newline at end of file diff --git a/gengine/tenant/static/admin.js b/gengine/app/static/admin.js similarity index 100% rename from gengine/tenant/static/admin.js rename to gengine/app/static/admin.js diff --git a/gengine/tenant/static/admin_layout.css b/gengine/app/static/admin_layout.css similarity index 100% rename from gengine/tenant/static/admin_layout.css rename to gengine/app/static/admin_layout.css diff --git a/gengine/tenant/static/api.js b/gengine/app/static/api.js similarity index 100% rename from gengine/tenant/static/api.js rename to gengine/app/static/api.js diff --git a/gengine/tenant/templates/admin/index.html b/gengine/app/templates/admin/index.html similarity index 100% rename from gengine/tenant/templates/admin/index.html rename to gengine/app/templates/admin/index.html diff --git a/gengine/olymp/templates/admin/layout.html b/gengine/app/templates/admin/layout.html similarity index 100% rename from gengine/olymp/templates/admin/layout.html rename to gengine/app/templates/admin/layout.html diff --git a/gengine/tenant/templates/admin_layout.html b/gengine/app/templates/admin_layout.html similarity index 100% rename from gengine/tenant/templates/admin_layout.html rename to gengine/app/templates/admin_layout.html diff --git a/gengine/tenant/templates/admin_maintenance.html b/gengine/app/templates/admin_maintenance.html similarity index 100% rename from gengine/tenant/templates/admin_maintenance.html rename to gengine/app/templates/admin_maintenance.html diff --git a/gengine/olymp/templates/error.html b/gengine/app/templates/error.html similarity index 100% rename from gengine/olymp/templates/error.html rename to gengine/app/templates/error.html diff --git a/gengine/tenant/views.py b/gengine/app/views.py similarity index 96% rename from gengine/tenant/views.py rename to gengine/app/views.py index a797048..e7428af 100644 --- a/gengine/tenant/views.py +++ b/gengine/app/views.py @@ -12,10 +12,9 @@ from pyramid.wsgi import wsgiapp2 from werkzeug import DebuggedApplication -from gengine.olymp.admin import olympadminapp -from gengine.tenant.admin import tenantadminapp -from gengine.tenant.formular import FormularEvaluationException -from gengine.tenant.model import ( +from gengine.app.admin import adminapp +from gengine.app.formular import FormularEvaluationException +from gengine.app.model import ( User, Achievement, Value, @@ -250,4 +249,4 @@ def generate(): @view_config(route_name='admin_tenant') @wsgiapp2 def admin_tenant(environ, start_response): - return HTTPSProxied(DebuggedApplication(tenantadminapp.wsgi_app, True))(environ, start_response) + return HTTPSProxied(DebuggedApplication(adminapp.wsgi_app, True))(environ, start_response) diff --git a/gengine/base/cache.py b/gengine/base/cache.py index 5aba16a..38cf4c2 100644 --- a/gengine/base/cache.py +++ b/gengine/base/cache.py @@ -3,7 +3,7 @@ from pyramid_dogpile_cache import get_region from gengine.base.context import get_context -def my_key_mangler(prefix, is_tenant_cache): +def my_key_mangler(prefix): def s(o): if type(o) == dict: return "_".join(["%s=%s" % (str(k), str(v)) for k, v in o.items()]) @@ -16,17 +16,13 @@ def s(o): def generate_key(key): ret = "" - if is_tenant_cache: - ret += get_context().tenant.id + "_" - else: - ret += "global_" ret += prefix + s(key).replace(" ", "") return ret return generate_key -def create_cache(name, is_tenant_cache=True): +def create_cache(name): ch = None try: @@ -36,12 +32,10 @@ def create_cache(name, is_tenant_cache=True): ch = make_region().configure('dogpile.cache.memory') warnings.warn("Warning: cache objects are in memory, are you creating docs?") - ch.key_mangler = my_key_mangler(name, is_tenant_cache) + ch.key_mangler = my_key_mangler(name) return ch - - cache_general = create_cache("general") cache_achievement_eval = create_cache("achievement_eval") cache_achievements_by_user_for_today = create_cache("achievements_by_user_for_today") diff --git a/gengine/base/model.py b/gengine/base/model.py index bbf3853..31bb28a 100644 --- a/gengine/base/model.py +++ b/gengine/base/model.py @@ -1,5 +1,6 @@ import pytz from pytz.exceptions import UnknownTimeZoneError +from sqlalchemy.inspection import inspect from sqlalchemy.sql.expression import select from sqlalchemy.sql.functions import func from zope.sqlalchemy.datamanager import mark_changed @@ -7,8 +8,16 @@ from gengine.metadata import DBSession from gengine.base.cache import cache_general +class ABaseMeta(type): + def __init__(cls, name, bases, nmspc): + super(ABaseMeta, cls).__init__(name, bases, nmspc) -class ABase(object): + def __getattr__(cls, item): + if item == "__table__": + return inspect(cls).local_table + raise AttributeError(item) + +class ABase(object, metaclass=ABaseMeta): """abstract base class which introduces a nice constructor for the model classes.""" def __init__(self, *args, **kw): @@ -24,7 +33,6 @@ def __str__(self): if hasattr(self, "__unicode__"): return self.__unicode__() - def calc_distance(latlong1, latlong2): """generates a sqlalchemy expression for distance query in km diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index 155d7de..c289401 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -58,16 +58,11 @@ def main(argv=sys.argv): if options.get("reset_db",False): Base.metadata.drop_all(engine) - engine.execute("DROP SCHEMA IF EXISTS olymp CASCADE") + engine.execute("DROP SCHEMA IF EXISTS public CASCADE") - tenant_schemas = engine.execute("SELECT DISTINCT schemaname FROM pg_catalog.pg_tables WHERE schemaname LIKE 't_%%'") + engine.execute("CREATE SCHEMA IF NOT EXISTS public") - for t in tenant_schemas: - engine.execute("DROP SCHEMA %s CASCADE" % (t["schemaname"],)) - - engine.execute("CREATE SCHEMA IF NOT EXISTS olymp") - - from gengine.olymp import model + from gengine.app import model tables = [t for name, t in model.__dict__.items() if isinstance(t, Table)] Base.metadata.create_all(engine, tables=tables) @@ -79,22 +74,18 @@ def main(argv=sys.argv): alembic_cfg = Config(attributes={ 'engine' : engine, - 'schema' : 'olymp' + 'schema' : 'public' }) - alembic_cfg.set_main_option("script_location", "gengine/olymp/alembic") + alembic_cfg.set_main_option("script_location", "gengine/app/alembic") command.stamp(alembic_cfg, "head") if options.get("populate_demo",False): - with transaction.manager: - t = model.Tenant(id="demo") - DBSession.add(t) + populate_demo(DBSession) - populate_demo(DBSession, "demo") +def populate_demo(DBSession): -def populate_demo(DBSession, tenant_id): - - from gengine.tenant.model import ( + from gengine.app.model import ( Achievement, AchievementCategory, Goal, @@ -122,8 +113,6 @@ def add_translation(variable, lang, text): return tr with transaction.manager: - DBSession.execute("SET search_path TO t_%s" % tenant_id) - lang_de = Language(name="de") lang_en = Language(name="en") DBSession.add(lang_de) diff --git a/gengine/olymp/admin.py b/gengine/olymp/admin.py deleted file mode 100644 index f6360e0..0000000 --- a/gengine/olymp/admin.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- -import jinja2 -import os -import pkg_resources -from flask import Flask -from flask.ext.admin import Admin -from flask.ext.admin.contrib.sqla import ModelView -from flask.helpers import send_from_directory - -from gengine.metadata import DBSession -from gengine.olymp.model import Tenant - -olympadminapp = None -admin = None - -def resole_uri(uri): - from pyramid.path import PkgResourcesAssetDescriptor - pkg_name, path = uri.split(":", 1) - a = PkgResourcesAssetDescriptor(pkg_name, path) - absolute = a.abspath() # this is sometimes not absolute :-/ - absolute = os.path.abspath(absolute) # so we make it absolute - return absolute - - -def get_static_view(folder, olympadminapp): - folder = resole_uri(folder) - - def send_static_file(filename): - cache_timeout = olympadminapp.get_send_file_max_age(filename) - return send_from_directory(folder, filename, cache_timeout=cache_timeout) - - return send_static_file - - -def init_admin(urlprefix="", secret="fKY7kJ2xSrbPC5yieEjV", override_admin=None, override_olympadminapp=None): - global olympadminapp, admin - - if not override_olympadminapp: - olympadminapp = Flask(__name__) - olympadminapp.debug = True - olympadminapp.secret_key = secret - olympadminapp.config.update(dict( - PREFERRED_URL_SCHEME='https' - )) - else: - olympadminapp = override_olympadminapp - - # lets add our template directory - my_loader = jinja2.ChoiceLoader([ - olympadminapp.jinja_loader, - jinja2.FileSystemLoader(resole_uri("gengine:olymp/templates")), - ]) - olympadminapp.jinja_loader = my_loader - - olympadminapp.add_url_rule('/static_gengine/', - endpoint='static_gengine', - view_func=get_static_view('gengine:olymp/static', olympadminapp)) - - @olympadminapp.context_processor - def inject_version(): - return {"gamification_engine_version": pkg_resources.get_distribution("gamification-engine").version} - - if not override_admin: - admin = Admin(olympadminapp, - name = "Zeus at Olympia - Gamification Engine", - base_template = 'admin_layout.html', - url = urlprefix + "/admin" - ) - else: - admin = override_admin - - admin.add_view(ModelViewTenant(DBSession)) - -class ModelViewTenant(ModelView): - column_list = ('id',) - column_searchable_list = ('id',) - form_columns = ('id',) - fast_mass_delete = True - - def __init__(self, session, **kwargs): - super(ModelViewTenant, self).__init__(Tenant, session, **kwargs) - diff --git a/gengine/olymp/model.py b/gengine/olymp/model.py deleted file mode 100644 index 3c7a9a6..0000000 --- a/gengine/olymp/model.py +++ /dev/null @@ -1,46 +0,0 @@ -import sqlalchemy.types as ty -from alembic import command -from alembic.config import Config -from sqlalchemy import event -from sqlalchemy.orm import mapper -from sqlalchemy.sql.schema import Table, Column, MetaData - -from gengine.metadata import Base -from gengine.base.model import ABase - -OLYMP_SCHEMA = "olymp" - -t_tenants = Table("tenants", Base.metadata, - Column("id", ty.String(), primary_key=True), - schema = OLYMP_SCHEMA -) - - -class Tenant(ABase): - def __unicode__(self, *args, **kwargs): - return "(ID: %s)" % (self.id,) - - -mapper(Tenant, t_tenants) - - -@event.listens_for(Tenant, "after_insert") -def create_tenant_schema(mapper, connection, target): - tenant_meta = MetaData(bind=connection) - schema = "t_"+target.id - - connection.execute("CREATE SCHEMA IF NOT EXISTS "+schema) - connection.execute("SET search_path TO "+schema) - - from gengine.tenant import model as model_tenant - - tables = [t.tometadata(tenant_meta, schema=schema) for name, t in model_tenant.__dict__.items() if isinstance(t, Table)] - - tenant_meta.create_all(tables=tables) - - alembic_cfg = Config(attributes={ - 'engine' : connection, - 'schema' : schema - }) - alembic_cfg.set_main_option("script_location", "gengine/tenant/alembic") - command.stamp(alembic_cfg, "head") \ No newline at end of file diff --git a/gengine/olymp/route.py b/gengine/olymp/route.py deleted file mode 100644 index 3a65854..0000000 --- a/gengine/olymp/route.py +++ /dev/null @@ -1,3 +0,0 @@ - -def config_routes(config): - pass \ No newline at end of file diff --git a/gengine/olymp/templates/admin/index.html b/gengine/olymp/templates/admin/index.html deleted file mode 100644 index 194abf6..0000000 --- a/gengine/olymp/templates/admin/index.html +++ /dev/null @@ -1,37 +0,0 @@ -{% extends 'admin/master.html' %} -{% import 'admin/lib.html' as lib with context %} - -{% macro extra() %} - -{% endmacro %} - -{% block head %} - {{ super() }} -{% endblock %} - -{% block body %} -
      - {% for msg in msgs %} -
    • {{ msg }}
      • - {% endfor %} -
    - -

    Global Administration ( Olymp )

    - -
    - -
    - -
    -
    - -
    -
    -
    - -
    -{% endblock %} - -{% block tail %} - {{ super() }} -{% endblock %} diff --git a/gengine/olymp/templates/admin_layout.html b/gengine/olymp/templates/admin_layout.html deleted file mode 100644 index 95acd9d..0000000 --- a/gengine/olymp/templates/admin_layout.html +++ /dev/null @@ -1,12 +0,0 @@ -{% import 'admin/layout.html' as layout with context -%} -{% extends 'admin/base.html' %} - -{% block head_tail %} - {{ super() }} - -{% endblock %} - -{% block tail %} - {{ super() }} - -{% endblock %} diff --git a/gengine/olymp/views.py b/gengine/olymp/views.py deleted file mode 100644 index c13af1c..0000000 --- a/gengine/olymp/views.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- - -from pyramid.view import view_config -from pyramid.wsgi import wsgiapp2 -from werkzeug import DebuggedApplication - -from gengine.olymp.admin import olympadminapp - -from gengine.wsgiutil import HTTPSProxied - -@view_config(route_name='admin_olymp') -@wsgiapp2 -def admin_olymp(environ, start_response): - return HTTPSProxied(DebuggedApplication(olympadminapp.wsgi_app, True))(environ, start_response) diff --git a/gengine/resources.py b/gengine/resources.py deleted file mode 100644 index a12beeb..0000000 --- a/gengine/resources.py +++ /dev/null @@ -1,36 +0,0 @@ -from sqlalchemy.sql.expression import select - -from gengine.metadata import DBSession -from gengine.olymp.model import t_tenants -from gengine.base.context import get_context - -def root_factory(request): - return RootResource() - -class RootResource(): - def __getitem__(self, item): - get_context().tenant = None - DBSession.execute("SET search_path TO olymp") - if item=="t": - return TenantCollectionResource(self, "t") - raise KeyError() - -class TenantCollectionResource(): - def __init__(self, parent, name): - self.__name__ = name - self.__parent__ = parent - - def __getitem__(self, item): - q = select([t_tenants.c.id,], from_obj = t_tenants).where(t_tenants.c.id==item) - tenant = DBSession.execute(q).fetchone() - if tenant: - get_context().tenant = tenant - DBSession.execute("SET search_path TO t_%s" % (tenant["id"],)) - return TenantResource(self, tenant) - raise KeyError() - -class TenantResource(): - def __init__(self, parent, tenant): - self.__name__ = tenant["id"] - self.__parent__ = parent - self.model = tenant diff --git a/gengine/tenant/__init__.py b/gengine/tenant/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/gengine/tenant/alembic/README b/gengine/tenant/alembic/README deleted file mode 100644 index 98e4f9c..0000000 --- a/gengine/tenant/alembic/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file diff --git a/gengine/tenant/alembic/env.py b/gengine/tenant/alembic/env.py deleted file mode 100644 index 25332d4..0000000 --- a/gengine/tenant/alembic/env.py +++ /dev/null @@ -1,66 +0,0 @@ -from __future__ import with_statement - -from logging.config import fileConfig - -import os -from alembic import context - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. - -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -if config.config_file_name: - fileConfig(config.config_file_name) - -overrides = {} - -durl = os.environ.get("DATABASE_URL") #heroku -if durl: - config.set_main_option('sqlalchemy.url',durl) - -# add your model's MetaData object here -# for 'autogenerate' support -from gengine.metadata import init_session,init_declarative_base -init_session() -init_declarative_base() - -from gengine.metadata import Base -target_metadata = Base.metadata - -# target_metadata = None - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - -def run_migrations_online(): - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - - engine = config.attributes["engine"] - - schema = config.attributes["schema"] - - #connectable = create_engine(url, poolclass=pool.NullPool) - - with engine.connect() as connection: - - connection.execute("SET search_path TO "+schema) - - context.configure( - connection=connection, - target_metadata=target_metadata - ) - - with context.begin_transaction(): - context.run_migrations() - -run_migrations_online() diff --git a/gengine/tenant/alembic/script.py.mako b/gengine/tenant/alembic/script.py.mako deleted file mode 100644 index 43c0940..0000000 --- a/gengine/tenant/alembic/script.py.mako +++ /dev/null @@ -1,24 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" - -# revision identifiers, used by Alembic. -revision = ${repr(up_revision)} -down_revision = ${repr(down_revision)} -branch_labels = ${repr(branch_labels)} -depends_on = ${repr(depends_on)} - -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -def upgrade(): - ${upgrades if upgrades else "pass"} - - -def downgrade(): - ${downgrades if downgrades else "pass"} diff --git a/gengine/tenant/alembic/versions/DUMMY b/gengine/tenant/alembic/versions/DUMMY deleted file mode 100644 index 90a1d60..0000000 --- a/gengine/tenant/alembic/versions/DUMMY +++ /dev/null @@ -1 +0,0 @@ -... \ No newline at end of file diff --git a/gengine/tenant/route.py b/gengine/tenant/route.py deleted file mode 100644 index 9138adf..0000000 --- a/gengine/tenant/route.py +++ /dev/null @@ -1,9 +0,0 @@ - -def config_routes(config): - config.add_route('get_progress', '/t/{tenant}/progress/{user_id}', traverse="/t/{tenant}") - config.add_route('increase_value', '/t/{tenant}/increase_value/{variable_name}/{user_id}', traverse="/t/{tenant}") - config.add_route('increase_value_with_key', '/t/{tenant}/increase_value/{variable_name}/{user_id}/{key}', traverse="/t/{tenant}") - config.add_route('increase_multi_values', '/t/{tenant}/increase_multi_values', traverse="/t/{tenant}") - config.add_route('add_or_update_user', '/t/{tenant}/add_or_update_user/{user_id}', traverse="/t/{tenant}") - config.add_route('delete_user', '/t/{tenant}/delete_user/{user_id}', traverse="/t/{tenant}") - config.add_route('get_achievement_level', '/t/{tenant}/achievement/{achievement_id}/level/{level}', traverse="/t/{tenant}") \ No newline at end of file diff --git a/gengine/tenant/templates/admin/layout.html b/gengine/tenant/templates/admin/layout.html deleted file mode 100644 index 12e3f9a..0000000 --- a/gengine/tenant/templates/admin/layout.html +++ /dev/null @@ -1,81 +0,0 @@ -{% macro menu_icon(item) -%} -{% set icon_type = item.get_icon_type() %} -{%- if icon_type %} - {% set icon_value = item.get_icon_value() %} - {% if icon_type == 'glyph' %} - - {% elif icon_type == 'image' %} - menu image - {% elif icon_type == 'image-url' %} - menu image - {% endif %} -{% endif %} -{%- endmacro %} - -{% macro menu() %} - {%- for item in admin_view.admin.menu() %} - {%- if item.is_category() -%} - {% set children = item.get_children() %} - {%- if children %} - {% set class_name = item.get_class_name() %} - {%- if item.is_active(admin_view) %} -
  • - {% else -%} -
  • - {%- endif %} - {{ menu_icon(item) }}{{ item.name }} -
      - {%- for child in children -%} - {% set class_name = child.get_class_name() %} - {%- if child.is_active(admin_view) %} -
    • - {% else %} - - {%- endif %} - {{ menu_icon(child) }}{{ child.name }} -
      • - {%- endfor %} -
    -
    • - {% endif %} - {%- else %} - {%- if item.is_accessible() and item.is_visible() -%} - {% set class_name = item.get_class_name() %} - {%- if item.is_active(admin_view) %} -
  • - {%- else %} - - {%- endif %} - {{ menu_icon(item) }}{{ item.name }} -
    • - {%- endif -%} - {% endif -%} - {% endfor %} -{% endmacro %} - -{% macro menu_links() %} - {% for item in admin_view.admin.menu_links() %} - {% if item.is_accessible() and item.is_visible() %} -
  • - {{ menu_icon(item) }}{{ item.name }} -
    • - {% endif %} - {% endfor %} -{% endmacro %} - -{% macro messages() %} - {% with messages = get_flashed_messages(with_categories=True) %} - {% if messages %} - {% for category, m in messages %} - {% if category %} -
    - {% else %} -
    - {% endif %} - x - {{ m }} -
    - {% endfor %} - {% endif %} - {% endwith %} -{% endmacro %} diff --git a/gengine/tenant/templates/error.html b/gengine/tenant/templates/error.html deleted file mode 100644 index 380d49c..0000000 --- a/gengine/tenant/templates/error.html +++ /dev/null @@ -1,20 +0,0 @@ - - - - - Error - - -
    -
    -

    - - ${message} -

    -

    - ${description} -

    -
    -
    - - From c130b1f03a35015ea93db4e7f699a23fb9f27e6f Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 24 Jun 2016 18:43:59 +0200 Subject: [PATCH 017/176] include ability to do alembic upgrade in init scripts --- gengine/maintenance/scripts/initializedb.py | 24 ++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index c289401..5a56bc6 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -62,13 +62,6 @@ def main(argv=sys.argv): engine.execute("CREATE SCHEMA IF NOT EXISTS public") - from gengine.app import model - - tables = [t for name, t in model.__dict__.items() if isinstance(t, Table)] - Base.metadata.create_all(engine, tables=tables) - - # then, load the Alembic configuration and generate the - # version table, "stamping" it with the most recent rev: from alembic.config import Config from alembic import command @@ -78,10 +71,21 @@ def main(argv=sys.argv): }) alembic_cfg.set_main_option("script_location", "gengine/app/alembic") - command.stamp(alembic_cfg, "head") + do_upgrade = options.get("upgrade",False) + if not do_upgrade: + #init + from gengine.app import model + + tables = [t for name, t in model.__dict__.items() if isinstance(t, Table)] + Base.metadata.create_all(engine, tables=tables) + + command.stamp(alembic_cfg, "head") - if options.get("populate_demo",False): - populate_demo(DBSession) + if options.get("populate_demo", False): + populate_demo(DBSession) + else: + #upgrade + command.upgrade(alembic_cfg,'head') def populate_demo(DBSession): From abbdd7633e1d9b3a6edabfb9e9037e23e53fcf2c Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 27 Jun 2016 13:12:52 +0200 Subject: [PATCH 018/176] script path --- gengine/maintenance/scripts/initializedb.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index 5a56bc6..a17012b 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -69,7 +69,11 @@ def main(argv=sys.argv): 'engine' : engine, 'schema' : 'public' }) - alembic_cfg.set_main_option("script_location", "gengine/app/alembic") + script_location = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'app/alembic' + ) + alembic_cfg.set_main_option("script_location", script_location) do_upgrade = options.get("upgrade",False) if not do_upgrade: From e1a0d53b5f7b7bfe82952342e3e6452a1491aca3 Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 27 Jun 2016 13:14:12 +0200 Subject: [PATCH 019/176] script path --- gengine/maintenance/scripts/initializedb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index a17012b..eb66621 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -70,7 +70,7 @@ def main(argv=sys.argv): 'schema' : 'public' }) script_location = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'app/alembic' ) alembic_cfg.set_main_option("script_location", script_location) From 109343e071399d11de76a5fc95ef6411c0c8cbb1 Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 27 Jun 2016 13:29:17 +0200 Subject: [PATCH 020/176] script path --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 9b217cb..d31e3fd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include *.txt *.ini *.cfg *.rst -recursive-include gengine *.ico *.png *.css *.gif *.jpg *.pt *.txt *.mak *.mako *.js *.html *.xml README alembic alembic/versions/*.py alembic/env.py +recursive-include gengine *.ico *.png *.css *.gif *.jpg *.pt *.txt *.mak *.mako *.js *.html *.xml README DUMMY alembic alembic/versions/*.py alembic/env.py recursive-include gengine_quickstart_template *.ini From 78c84aac2a670d09918d4a129c14da362a587314 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 8 Jul 2016 12:33:15 +0200 Subject: [PATCH 021/176] gitignore --- .gitignore | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 772bee4..ec4ccb3 100644 --- a/.gitignore +++ b/.gitignore @@ -60,4 +60,10 @@ target/ *.pydevproject /*.dbm -venv/ \ No newline at end of file +venv/ +# Elastic Beanstalk Files +.elasticbeanstalk/* +!.elasticbeanstalk/*.cfg.yml +!.elasticbeanstalk/*.global.yml + +.idea From 4ad2f6bcc1b7c5e1ea18647e53ac4e89c7a14806 Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 11 Jul 2016 09:43:47 +0200 Subject: [PATCH 022/176] seperate actual init code --- gengine/maintenance/scripts/initializedb.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index eb66621..b0eec4f 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -36,7 +36,10 @@ def main(argv=sys.argv): murl = os.environ.get("MEMCACHED_URL") if murl: settings['urlcache_url']=murl - + + initialize(settings,options) + +def initialize(settings,options): engine = engine_from_config(settings, 'sqlalchemy.') config = Configurator(settings=settings) @@ -91,6 +94,8 @@ def main(argv=sys.argv): #upgrade command.upgrade(alembic_cfg,'head') + engine.dispose() + def populate_demo(DBSession): from gengine.app.model import ( From e37917dd7153b59cd0851ac67b2108205e7ce872 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 14 Jul 2016 14:37:59 +0200 Subject: [PATCH 023/176] start to implement user authentication --- gengine/__init__.py | 31 +++++++++++- gengine/app/admin.py | 30 +++++++++-- gengine/app/model.py | 118 ++++++++++++++++++++++++++++++++++++++++++- gengine/app/route.py | 4 +- gengine/app/views.py | 46 +++++++++++++++-- setup.py | 3 +- 6 files changed, 220 insertions(+), 12 deletions(-) diff --git a/gengine/__init__.py b/gengine/__init__.py index 0dc1a69..3712fef 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -2,6 +2,7 @@ from pyramid.events import NewRequest from gengine.base.context import reset_context +from gengine.base.errors import APIError __version__ = '0.2.0' @@ -56,7 +57,35 @@ def reset_context_on_new_request(event): urlcache_url = settings.get("urlcache_url","127.0.0.1:11211") urlcache_active = asbool(os.environ.get("URLCACHE_ACTIVE", settings.get("urlcache_active",True))) - + + #auth + def get_user(request): + token = request.headers.get('token') + if token is not None: + from gengine.app.model import DBSession, AuthUser, AuthToken + tokenObj = DBSession.query(AuthToken).filter(AuthToken.token==token).first() + user = None + if tokenObj and tokenObj.valid_until Date: Thu, 14 Jul 2016 18:30:52 +0200 Subject: [PATCH 024/176] integrate user authorization into views, admin --- gengine/__init__.py | 28 ++- gengine/app/admin.py | 4 + gengine/app/model.py | 85 ++++--- gengine/app/permissions.py | 23 ++ gengine/app/static/api.js | 4 +- gengine/app/templates/admin/index.html | 3 +- gengine/app/views.py | 233 ++++++++++++++------ gengine/base/cache.py | 64 ------ gengine/base/settings.py | 9 + gengine/maintenance/scripts/initializedb.py | 18 +- 10 files changed, 297 insertions(+), 174 deletions(-) create mode 100644 gengine/app/permissions.py create mode 100644 gengine/base/settings.py diff --git a/gengine/__init__.py b/gengine/__init__.py index 3712fef..701bce5 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -3,6 +3,7 @@ from gengine.base.context import reset_context from gengine.base.errors import APIError +from gengine.base.settings import set_settings __version__ = '0.2.0' @@ -20,6 +21,9 @@ def main(global_config, **settings): """ This function returns a Pyramid WSGI application. """ + + set_settings(settings) + engine = engine_from_config(settings, 'sqlalchemy.', connect_args={"options": "-c timezone=utc"}, ) from gengine.metadata import init_session, init_declarative_base, init_db @@ -60,7 +64,9 @@ def reset_context_on_new_request(event): #auth def get_user(request): - token = request.headers.get('token') + if not asbool(settings.get("enable_user_authentication",False)): + return None + token = request.headers.get('X-Auth-Token') if token is not None: from gengine.app.model import DBSession, AuthUser, AuthToken tokenObj = DBSession.query(AuthToken).filter(AuthToken.token==token).first() @@ -77,32 +83,32 @@ def get_user(request): return None def get_permissions(request): + if not asbool(settings.get("enable_user_authentication", False)): + return [] from gengine.app.model import DBSession, t_auth_tokens, t_auth_users, t_auth_roles, t_auth_roles_permissions, t_auth_users_roles from sqlalchemy.sql import select j = t_auth_tokens.join(t_auth_users).join(t_auth_users_roles).join(t_auth_roles).join(t_auth_roles_permissions) - q = select([t_auth_roles_permissions.c.name],from_obj=j).where(t_auth_tokens.c.token==request.headers.get("AUTH_TOKEN")) + q = select([t_auth_roles_permissions.c.name],from_obj=j).where(t_auth_tokens.c.token==request.headers.get("X-Auth-Token")) return [r["name"] for r in DBSession.execute(q).fetchall()] + def has_perm(request, name): + return name in request.permissions + config.add_request_method(get_user, 'user', reify=True) config.add_request_method(get_permissions, 'permissions', reify=True) + config.add_request_method(has_perm, 'has_perm') #routes - from gengine.app.route import config_routes as config_tenant_routes + from gengine.app.route import config_routes as config_app_routes - config.include(config_tenant_routes, route_prefix=urlprefix) + config.include(config_app_routes, route_prefix=urlprefix) - config.add_route('admin_tenant', '/*subpath') #prefix is set in flaskadmin.py + config.add_route('admin_app', '/*subpath') #prefix is set in flaskadmin.py from gengine.app.admin import init_admin as init_tenantadmin init_tenantadmin(urlprefix=urlprefix, secret=settings.get("flaskadmin_secret","fKY7kJ2xSrbPC5yieEjV")) - from gengine.base.cache import setup_urlcache - setup_urlcache(prefix=urlprefix, - url = urlcache_url, - active = urlcache_active, - id = urlcacheid) - #date serialization json_renderer = JSON() def datetime_adapter(obj, request): diff --git a/gengine/app/admin.py b/gengine/app/admin.py index 8138ab5..46eef0d 100644 --- a/gengine/app/admin.py +++ b/gengine/app/admin.py @@ -16,6 +16,7 @@ from gengine.app.model import DBSession, Variable, Goal, AchievementCategory, Achievement, AchievementProperty, GoalProperty, AchievementAchievementProperty, AchievementReward,\ GoalGoalProperty, Reward, User, GoalEvaluationCache, Value, AchievementUser, TranslationVariable, Language, Translation, \ AuthUser, AuthRole, AuthRolePermission +from gengine.app.permissions import yield_all_perms adminapp=None admin=None @@ -233,6 +234,9 @@ def __init__(self, session, **kwargs): class PermissionInlineModelForm(InlineFormAdmin): form_columns = ('id','name') + form_choices = { + "name" : sorted(list(yield_all_perms()),key=lambda x:x[1]) + } class ModelViewAuthRole(ModelView): column_list = ('id', 'name', 'permissions') diff --git a/gengine/app/model.py b/gengine/app/model.py index 8a0bd89..cc07f40 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -7,12 +7,14 @@ import hashlib import pytz import sqlalchemy.types as ty +from pyramid.settings import asbool from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.sql.schema import UniqueConstraint +from gengine.app.permissions import perm_global_increase_value from gengine.base.model import ABase, exists_by_expr, datetime_trunc, calc_distance, coords, update_connection from gengine.base.cache import cache_general, cache_goal_evaluation, cache_achievement_eval, cache_achievements_users_levels, \ - cache_achievements_by_user_for_today, invalidate, cache_goal_statements, cache_translations + cache_achievements_by_user_for_today, cache_goal_statements, cache_translations from sqlalchemy import ( Table, ForeignKey, @@ -30,6 +32,7 @@ ) from sqlalchemy.sql import bindparam +from gengine.base.settings import get_settings from gengine.metadata import Base, DBSession from gengine.app.formular import evaluate_condition, evaluate_value_expression, evaluate_string @@ -43,11 +46,14 @@ Column('created_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow), ) +def get_default_token_valid_time(): + return datetime.datetime.utcnow() + datetime.timedelta(days=30) + t_auth_tokens = Table("auth_tokens", Base.metadata, Column("id", ty.BigInteger, primary_key=True), - Column("user_id", ty.BigInteger, ForeignKey("auth_users.id"), nullable=False), + Column("user_id", ty.BigInteger, ForeignKey("auth_users.id", ondelete="CASCADE"), nullable=False), Column("token", ty.String, nullable=False), - Column('valid_until', ty.DateTime, nullable = False, default=lambda: datetime.datetime.utcnow()+datetime.timedelta.days(30)), + Column('valid_until', ty.DateTime, nullable = False, default=get_default_token_valid_time), ) t_auth_roles = Table("auth_roles", Base.metadata, @@ -56,15 +62,13 @@ ) t_auth_users_roles = Table("auth_users_roles", Base.metadata, - Column("id", ty.BigInteger, primary_key=True, nullable=False), - Column("user_id", ty.BigInteger, ForeignKey("auth_users.id"), primary_key=True, nullable=False), - Column("role_id", ty.BigInteger, ForeignKey("auth_roles.id"), primary_key=True, nullable=False), - UniqueConstraint("user_id","role_id") + Column("user_id", ty.BigInteger, ForeignKey("auth_users.id", ondelete="CASCADE"), primary_key=True, nullable=False), + Column("role_id", ty.BigInteger, ForeignKey("auth_roles.id", ondelete="CASCADE"), primary_key=True, nullable=False), ) t_auth_roles_permissions = Table("auth_roles_permissions", Base.metadata, Column("id", ty.Integer, primary_key=True), - Column("role_id", ty.Integer, ForeignKey("auth_roles.id", use_alter=True), nullable=False, index=True), + Column("role_id", ty.Integer, ForeignKey("auth_roles.id", use_alter=True, ondelete="CASCADE"), nullable=False, index=True), Column("name", ty.String(255), nullable=False), UniqueConstraint("role_id", "name") ) @@ -112,6 +116,7 @@ Column("max_distance", ty.Integer, nullable=True), Column('priority', ty.Integer, index=True, default=0), Column('relevance',ty.Enum("friends","city","own", name="relevance_types"), default="own"), + Column('view_permission',ty.Enum("everyone", "own", name="achievement_view_permission"), default="everyone"), ) t_goals = Table("goals", Base.metadata, @@ -142,6 +147,7 @@ Column('id', ty.Integer, primary_key = True), Column('name', ty.String(255), nullable = False, index=True), Column('group', ty.Enum("year","month","week","day","none", name="variable_group_types"), nullable = False, default="none"), + Column('increase_permission',ty.Enum("own", "admin", name="variable_increase_permission"), default="admin"), ) t_values = Table('values', Base.metadata, @@ -235,12 +241,13 @@ def password(self): @password.setter def password(self,new_pw): - import argon2 - import crypt - import base64 - self.password_salt = crypt.mksalt() - hash = argon2.argon2_hash(new_pw, self.password_salt) - self.password_hash = base64.b64encode(hash).decode("UTF-8") + if new_pw!=self.password_hash: + import argon2 + import crypt + import base64 + self.password_salt = crypt.mksalt() + hash = argon2.argon2_hash(new_pw, self.password_salt) + self.password_hash = base64.b64encode(hash).decode("UTF-8") def verify_password(self, pw): import argon2 @@ -250,6 +257,21 @@ def verify_password(self, pw): is_valid = check == orig return is_valid + def get_or_create_token(self): + tokenObj = DBSession.query(AuthToken).filter(AuthToken.valid_until>=datetime.datetime.utcnow()).first() + + if not tokenObj: + token = AuthToken.generate_token() + tokenObj = AuthToken( + user_id=self.id, + token=token + ) + + DBSession.add(tokenObj) + + return tokenObj + + class AuthToken(ABase): @staticmethod @@ -266,11 +288,11 @@ def __unicode__(self, *args, **kwargs): class AuthRole(ABase): def __unicode__(self, *args, **kwargs): - return "Role %s" % (self.id,) + return "Role %s" % (self.name,) -class AuthUserRole(ABase): - def __unicode__(self, *args, **kwargs): - return "UserRole %s" % (self.id,) +#class AuthUserRole(ABase): +# def __unicode__(self, *args, **kwargs): +# return "UserRole %s" % (self.id,) class AuthRolePermission(ABase): def __unicode__(self, *args, **kwargs): @@ -469,6 +491,19 @@ def invalidate_caches_for_variable_and_user(cls,variable_id,user_id): for entry in goalsandachievements: Achievement.invalidate_evaluate_cache(user_id,entry["achievement"]) + @classmethod + def may_increase(cls, variable_row, request, user_id): + if not asbool(get_settings().get("enable_user_authentication", False)): + #Authentication deactivated + return True + if request.has_perm(perm_global_increase_value): + # I'm the global admin + return True + if variable_row["increase_permission"]=="own" and request.user and str(request.user.id)==str(user_id): + #The variable may be updated for myself + return True + return False + class Value(ABase): """A Value describes the relation of the user to a variable. @@ -641,6 +676,7 @@ def basic_output(cls,achievement,goals,include_levels=True, out = { "id" : achievement["id"], + "view_permission" : achievement["view_permission"], "internal_name" : achievement["name"], "maxlevel" : achievement["maxlevel"], "priority" : achievement["priority"], @@ -787,7 +823,6 @@ def invalidate_evaluate_cache(cls,user_id,achievement): #We neeed to invalidate for all relevant users because of the leaderboards for uid in Achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user_id): cache_achievement_eval.delete("%s_%s" % (uid,achievement["id"])) - invalidate("/progress/"+str(uid)) @classmethod @cache_general.cache_on_arguments() @@ -1241,17 +1276,17 @@ def get_languages(cls): return DBSession.execute(t_languages.select()).fetchall() mapper(AuthUser, t_auth_users, properties={ - + 'roles' : relationship(AuthRole, secondary=t_auth_users_roles, backref="users") }) mapper(AuthToken, t_auth_tokens, properties={ - + 'user' : relationship(AuthUser, backref="tokens") }) -mapper(AuthUserRole, t_auth_users_roles, properties={ - 'user' : relationship(AuthUser, backref="roles"), - 'role' : relationship(AuthRole, backref="users") -}) +#mapper(AuthUserRole, t_auth_users_roles, properties={ +# 'user' : relationship(AuthUser, backref="roles"), +# 'role' : relationship(AuthRole, backref="users") +#}) mapper(AuthRole, t_auth_roles, properties={ diff --git a/gengine/app/permissions.py b/gengine/app/permissions.py new file mode 100644 index 0000000..adfd5c5 --- /dev/null +++ b/gengine/app/permissions.py @@ -0,0 +1,23 @@ + +perm_global_access_admin_ui = "global_access_admin_ui" +desc_global_access_admin_ui = "(Admin) Can access Admin-UI" + +perm_global_update_user_infos = "global_update_user_infos" +desc_global_update_user_infos = "(Admin) Update every user's information" + +perm_own_update_user_infos = "own_update_user_infos" +desc_own_update_user_infos = "Update my own infos" + +perm_global_delete_user = "global_delete_user" +desc_global_delete_user = "(Admin) Delete all users" + +perm_own_delete_user = "own_delete_user" +desc_own_delete_user = "Delete myself" + +perm_global_increase_value = "global_increase_value" +desc_global_increase_value = "(Admin) Increase every user's values" + +def yield_all_perms(): + for k,v in globals().items(): + if k.startswith("perm_"): + yield (v, globals().get("desc_"+k.lstrip("perm_"),k)) diff --git a/gengine/app/static/api.js b/gengine/app/static/api.js index 85fcb26..e9371eb 100644 --- a/gengine/app/static/api.js +++ b/gengine/app/static/api.js @@ -33,8 +33,10 @@ var setupAPIForm = function($, defaultcall, fields, api_funcs) { "url" : "/", "data" : { - } + }, }; + $.ajaxSetup({headers: {"X-Auth-Token": $.cookie("X-Auth-Token")}}); + var api_settings_url; var api_settings_method; var api_settings_postparams; diff --git a/gengine/app/templates/admin/index.html b/gengine/app/templates/admin/index.html index 78ebc0b..f5fef58 100644 --- a/gengine/app/templates/admin/index.html +++ b/gengine/app/templates/admin/index.html @@ -159,4 +159,5 @@

    Welcome to the Gamification Engine Admin-Area

    {% block tail %} {{ super() }} -{% endblock %} + +{% endblock %} \ No newline at end of file diff --git a/gengine/app/views.py b/gengine/app/views.py index c362f31..f8e0655 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -1,10 +1,20 @@ # -*- coding: utf-8 -*- import traceback +import binascii +from http.cookies import SimpleCookie + +import base64 import copy +import datetime +from pyramid.request import Request +from pyramid.response import Response +from pyramid.settings import asbool +from sqlalchemy.sql.expression import select -from gengine.base.model import valid_timezone -from gengine.base.cache import get_or_set, set_value +from gengine.app.permissions import perm_own_update_user_infos, perm_global_update_user_infos, perm_global_delete_user, perm_own_delete_user, \ + perm_global_access_admin_ui +from gengine.base.model import valid_timezone, exists_by_expr from gengine.base.errors import APIError from pyramid.exceptions import NotFound from pyramid.renderers import render @@ -19,19 +29,27 @@ Achievement, Value, Variable, - AuthUser, AuthToken) + AuthUser, AuthToken, t_users, t_auth_users, t_auth_users_roles, t_auth_roles, t_auth_roles_permissions) +from gengine.base.settings import get_settings from gengine.metadata import DBSession from gengine.wsgiutil import HTTPSProxied -def require_perm(request, perm): - pass - @view_config(route_name='add_or_update_user', renderer='string', request_method="POST") def add_or_update_user(request): """add a user and set its metadata""" - + user_id = int(request.matchdict["user_id"]) - + + if asbool(get_settings().get("enable_user_authentication", False)): + #ensure that the user exists and we have the permission to update it + may_update = request.has_perm(perm_global_update_user_infos) or request.has_perm(perm_own_update_user_infos) and request.user.id == user_id + if not may_update: + raise APIError(403, "forbidden", "You may not edit this user.") + + if not exists_by_expr(t_users,t_users.c.id==user_id): + raise APIError(403, "forbidden", "The user does not exist. As the user authentication is enabled, you need to create the AuthUser first.") + + lat=None if len(request.POST.get("lat",""))>0: lat = float(request.POST["lat"]) @@ -82,49 +100,55 @@ def add_or_update_user(request): @view_config(route_name='delete_user', renderer='string', request_method="DELETE") def delete_user(request): """delete a user completely""" - user_id = int(request.matchdict["user_id"]) + + if asbool(get_settings().get("enable_user_authentication", False)): + # ensure that the user exists and we have the permission to update it + may_delete = request.has_perm(perm_global_delete_user) or request.has_perm(perm_own_delete_user) and request.user.id == user_id + if not may_delete: + raise APIError(403, "forbidden", "You may not delete this user.") + User.delete_user(user_id) return {"status" : "OK"} -def _get_progress(user,force_generation=False): - def generate(): - achievements = Achievement.get_achievements_by_user_for_today(user) - - def ea(achievement): - try: - #print "evaluating "+`achievement["id"]` - return Achievement.evaluate(user, achievement["id"]) - except FormularEvaluationException as e: - return { "error": "Cannot evaluate formular: " + e.message, "id" : achievement["id"] } - except Exception as e: - tb = traceback.format_exc() - return { "error": tb, "id" : achievement["id"] } - - check = lambda x : x!=None and not "error" in x and (x["hidden"]==False or x["level"]>0) - - evaluatelist = [ea(achievement) for achievement in achievements] - - ret = { - "achievements" : { - x["id"] : x for x in evaluatelist if check(x) - }, - "achievement_errors" : { - x["id"] : x for x in evaluatelist if x!=None and "error" in x - } +def _get_progress(achievements_for_user, requesting_user): + + achievements = Achievement.get_achievements_by_user_for_today(achievements_for_user) + + def ea(achievement): + try: + return Achievement.evaluate(achievements_for_user, achievement["id"]) + except FormularEvaluationException as e: + return { "error": "Cannot evaluate formular: " + e.message, "id" : achievement["id"] } + except Exception as e: + tb = traceback.format_exc() + return { "error": tb, "id" : achievement["id"] } + + check = lambda x : x!=None and not "error" in x and (x["hidden"]==False or x["level"]>0) + + def may_view(achievement, requesting_user): + if not asbool(get_settings().get("enable_user_authentication", False)): + return True + + if achievement["view_permission"] == "everyone": + return True + if achievement["view_permission"] == "own" and achievements_for_user["id"] == requesting_user["id"]: + return True + return False + + evaluatelist = [ea(achievement) for achievement in achievements if may_view(achievement, requesting_user)] + + ret = { + "achievements" : { + x["id"] : x for x in evaluatelist if check(x) + }, + "achievement_errors" : { + x["id"] : x for x in evaluatelist if x!=None and "error" in x } - - return render("json",ret),ret - - key = "/progress/"+str(user.id) - - if not force_generation: - #in this case, we do not return the decoded json object - the caller has to take of this if needed - return get_or_set(key,lambda:generate()[0]), None - else: - ret_str, ret = generate() - set_value(key,ret_str) - return ret_str, ret + } + + return ret + @view_config(route_name='get_progress', renderer='string') def get_progress(request): @@ -136,7 +160,7 @@ def get_progress(request): raise NotFound("user not found") request.response.content_type = "application/json" - progress = _get_progress(user, force_generation=False) + progress = _get_progress(achievements_for_user=user, requesting_user=request.user) json_string, pmap = progress return json_string @@ -161,10 +185,14 @@ def increase_value(request): variable = Variable.get_variable_by_name(variable_name) if not variable: raise APIError(404, "variable_not_found", "variable not found") + + if asbool(get_settings().get("enable_user_authentication", False)): + if not Variable.may_increase(variable, request, user_id): + raise APIError(403, "forbidden", "You may not increase the variable for this user.") Value.increase_value(variable_name, user, value, key) - output = copy.deepcopy(_get_progress(user,force_generation=True)[1]) #1 is the map + output = _get_progress(achievements_for_user=user, requesting_user=request.user) for aid in list(output["achievements"].keys()): if len(output["achievements"][aid]["new_levels"])>0: if "levels" in output["achievements"][aid]: @@ -192,6 +220,10 @@ def increase_multi_values(request): for variable_name, values_and_keys in values.items(): for value_and_key in values_and_keys: variable = Variable.get_variable_by_name(variable_name) + + if asbool(get_settings().get("enable_user_authentication", False)): + if not Variable.may_increase(variable, request, user_id): + raise APIError(403, "forbidden", "You may not increase the variable %s for user %s." % (variable_name, user_id)) if not variable: raise APIError(404, "variable_not_found", "variable %s not found" % (variable_name,)) @@ -203,8 +235,8 @@ def increase_multi_values(request): key = value_and_key.get('key','') Value.increase_value(variable_name, user, value, key) - - output = copy.deepcopy(_get_progress(user, force_generation=True)[1]) # 1 is the map + + output = _get_progress(achievements_for_user=user, requesting_user=request.user) for aid in list(output["achievements"].keys()): if len(output["achievements"][aid]["new_levels"])>0: @@ -222,7 +254,7 @@ def increase_multi_values(request): return ret -@view_config(route_name='get_achievement_level', renderer='string', request_method="GET") +@view_config(route_name='get_achievement_level', renderer='json', request_method="GET") def get_achievement_level(request): """get all information about an achievement for a specific level""" try: @@ -230,24 +262,19 @@ def get_achievement_level(request): level = int(request.matchdict.get("level",None)) except: raise APIError(400, "invalid_input", "invalid input") - - def generate(): - achievement = Achievement.get_achievement(achievement_id) - - if not achievement: - raise APIError(404, "achievement_not_found", "achievement not found") - - level_output = Achievement.basic_output(achievement, [], True, level).get("levels").get(str(level), {"properties":{},"rewards":{}}) - if "goals" in level_output: - del level_output["goals"] - if "level" in level_output: - del level_output["level"] - return render("json",level_output) - - key = "/achievement/"+str(achievement_id)+"/level/"+str(level) - request.response.content_type = 'application/json' - - return get_or_set(key,generate) + + achievement = Achievement.get_achievement(achievement_id) + + if not achievement: + raise APIError(404, "achievement_not_found", "achievement not found") + + level_output = Achievement.basic_output(achievement, [], True, level).get("levels").get(str(level), {"properties":{},"rewards":{}}) + if "goals" in level_output: + del level_output["goals"] + if "level" in level_output: + del level_output["level"] + + return level_output @view_config(route_name='auth_login', renderer='json', request_method="POST") @@ -284,7 +311,71 @@ def auth_login(request): "token" : token } -@view_config(route_name='admin_tenant') +@view_config(route_name='admin_app') @wsgiapp2 def admin_tenant(environ, start_response): - return HTTPSProxied(DebuggedApplication(adminapp.wsgi_app, True))(environ, start_response) + + def admin_app(): + return HTTPSProxied(DebuggedApplication(adminapp.wsgi_app, True))(environ, start_response) + + def request_auth(): + resp = Response() + resp.status_code = 401 + resp.www_authenticate = 'Basic realm="%s"' % ("Gamification Engine Admin",) + return resp(environ, start_response) + + if not asbool(get_settings().get("enable_user_authentication", False)): + return admin_app() + + req = Request(environ) + + def _get_basicauth_credentials(request): + authorization = request.headers.get("authorization","") + try: + authmeth, auth = authorization.split(' ', 1) + except ValueError: # not enough values to unpack + return None + if authmeth.lower() == 'basic': + try: + auth = base64.b64decode(auth.strip()).decode("UTF-8") + except binascii.Error: # can't decode + return None + try: + login, password = auth.split(':', 1) + except ValueError: # not enough values to unpack + return None + return {'login': login, 'password': password} + return None + + user = None + cred = _get_basicauth_credentials(req) + token = req.cookies.get("token",None) + if token: + tokenObj = DBSession.query(AuthToken).filter(AuthToken.token == token).first() + user = None + if tokenObj and tokenObj.valid_until < datetime.datetime.utcnow(): + tokenObj.extend() + if tokenObj: + user = tokenObj.user + + if not user: + if cred: + user = DBSession.query(AuthUser).filter_by(email=cred["login"]).first() + if not user or not user.verify_password(cred["password"]): + return request_auth() + + if user: + j = t_auth_users.join(t_auth_users_roles).join(t_auth_roles).join(t_auth_roles_permissions) + q = select([t_auth_roles_permissions.c.name], from_obj=j).where(t_auth_users.c.id==user.id) + permissions = [r["name"] for r in DBSession.execute(q).fetchall()] + if not perm_global_access_admin_ui in permissions: + return request_auth() + else: + cookie = SimpleCookie() + cookie['X-Auth-Token'] = user.get_or_create_token().token + cookie['X-Auth-Token']['path'] = get_settings().get("urlprefix","").rstrip("/")+"/" + + cookieheaders = ('Set-Cookie', cookie['X-Auth-Token'].OutputString()) + start_response(200,[cookieheaders,]) + + return admin_app() \ No newline at end of file diff --git a/gengine/base/cache.py b/gengine/base/cache.py index 38cf4c2..f19b326 100644 --- a/gengine/base/cache.py +++ b/gengine/base/cache.py @@ -1,7 +1,6 @@ import warnings from dogpile.cache import make_region from pyramid_dogpile_cache import get_region -from gengine.base.context import get_context def my_key_mangler(prefix): def s(o): @@ -53,66 +52,3 @@ def clear_all_caches(): cache_general.invalidate(hard=True) cache_goal_evaluation.invalidate(hard=True) cache_goal_statements.invalidate(hard=True) - invalidate_all_mc() - - -# URL Cache - -# -*- coding: utf-8 -*- -from pymemcache.client import Client - -host = "localhost" -port = 11211 -urlprefix = "" -is_active = True -urlcacheid = "gengine" - - -def setup_urlcache(prefix, url, active, id): - global urlprefix, host, port, is_active, urlcacheid - urlprefix = prefix - host, port = url.split(":") - port = int(port) - is_active = active - urlcacheid = id - - -def __build_key(key): - return "::URL_CACHE::" + str(urlcacheid) + "::" + urlprefix + str(key) - - -def get_or_set(key, generator): - if is_active: - client = Client((host, port)) - key = __build_key(key) - result = client.get(key) - if not result: - result = generator() - client.set(key, result) - client.quit() - return result - else: - return generator() - - -def set_value(key, value): - if is_active: - client = Client((host, port)) - key = __build_key(key) - client.set(key, value) - client.quit() - - -def invalidate(key): - if is_active: - key = __build_key(key) - client = Client((host, port)) - client.delete(key) - client.quit() - - -def invalidate_all_mc(): - if is_active: - client = Client((host, port)) - client.flush_all() - client.quit() \ No newline at end of file diff --git a/gengine/base/settings.py b/gengine/base/settings.py new file mode 100644 index 0000000..0e468a1 --- /dev/null +++ b/gengine/base/settings.py @@ -0,0 +1,9 @@ +_settings = None + +def set_settings(settings): + global _settings + _settings = settings + +def get_settings(): + global _settings + return _settings \ No newline at end of file diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index b0eec4f..53d9e6a 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -13,6 +13,8 @@ from sqlalchemy import engine_from_config from sqlalchemy.sql.schema import Table +from gengine.app.permissions import perm_global_delete_user, perm_global_increase_value, perm_global_update_user_infos + def usage(argv): cmd = os.path.basename(argv[0]) @@ -112,7 +114,10 @@ def populate_demo(DBSession): Reward, AchievementProperty, AchievementAchievementProperty, - AchievementReward + AchievementReward, + AuthUser, + AuthRole, + AuthRolePermission ) def add_translation_variable(name): @@ -241,5 +246,16 @@ def add_translation(variable, lang, text): DBSession.add(user2) DBSession.add(user3) + auth_user = AuthUser(id=1,email="admin@gamification-software.com",password="test123",active=True) + DBSession.add(auth_user) + + auth_role = AuthRole(name="Global Admin") + DBSession.add(auth_role) + + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_delete_user)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_increase_value)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_update_user_infos)) + + if __name__ == '__main__': main() \ No newline at end of file From 277c2064afd1783aaa0a5a3dac7902aa8dd77351 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 15 Jul 2016 09:38:15 +0200 Subject: [PATCH 025/176] - add device table to prepare push-functionality - auth_users has now a foreignkey to users table --- gengine/app/model.py | 49 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index cc07f40..cc670ed 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -37,8 +37,19 @@ from gengine.app.formular import evaluate_condition, evaluate_value_expression, evaluate_string +t_users = Table("users", Base.metadata, + Column('id', ty.BigInteger, primary_key = True), + Column("lat", ty.Float(Precision=64), nullable=True), + Column("lng", ty.Float(Precision=64), nullable=True), + Column("timezone", ty.String(), nullable=False, default="UTC"), + Column("country", ty.String(), nullable=True, default=None), + Column("region", ty.String(), nullable=True, default=None), + Column("city", ty.String(), nullable=True, default=None), + Column('created_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow), +) + t_auth_users = Table("auth_users", Base.metadata, - Column("id", ty.BigInteger, primary_key = True), + Column('user_id', ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), primary_key = True, nullable=False), Column("email", ty.String, unique=True), Column("password_hash", ty.String, nullable=False), Column("password_salt", ty.Unicode, nullable=False), @@ -51,7 +62,7 @@ def get_default_token_valid_time(): t_auth_tokens = Table("auth_tokens", Base.metadata, Column("id", ty.BigInteger, primary_key=True), - Column("user_id", ty.BigInteger, ForeignKey("auth_users.id", ondelete="CASCADE"), nullable=False), + Column("user_id", ty.BigInteger, ForeignKey("auth_users.user_id", ondelete="CASCADE"), nullable=False), Column("token", ty.String, nullable=False), Column('valid_until', ty.DateTime, nullable = False, default=get_default_token_valid_time), ) @@ -62,7 +73,7 @@ def get_default_token_valid_time(): ) t_auth_users_roles = Table("auth_users_roles", Base.metadata, - Column("user_id", ty.BigInteger, ForeignKey("auth_users.id", ondelete="CASCADE"), primary_key=True, nullable=False), + Column("user_id", ty.BigInteger, ForeignKey("auth_users.user_id", ondelete="CASCADE"), primary_key=True, nullable=False), Column("role_id", ty.BigInteger, ForeignKey("auth_roles.id", ondelete="CASCADE"), primary_key=True, nullable=False), ) @@ -73,15 +84,13 @@ def get_default_token_valid_time(): UniqueConstraint("role_id", "name") ) -t_users = Table("users", Base.metadata, +t_user_device = Table('user_devices', Base.metadata, Column('id', ty.BigInteger, primary_key = True), - Column("lat", ty.Float(Precision=64), nullable=True), - Column("lng", ty.Float(Precision=64), nullable=True), - Column("timezone", ty.String(), nullable=False, default="UTC"), - Column("country", ty.String(), nullable=True, default=None), - Column("region", ty.String(), nullable=True, default=None), - Column("city", ty.String(), nullable=True, default=None), - Column('created_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow), + Column('user_id', ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), primary_key = True, nullable=False), + Column('device_os', ty.String, nullable=False), + Column('push_id', ty.String(), nullable=False), + Column('app_version', ty.String(), nullable=False), + Column('registered_at', ty.DateTime(), nullable=False, default=datetime.datetime.utcnow), ) t_users_users = Table("users_users", Base.metadata, @@ -235,6 +244,11 @@ def get_default_token_valid_time(): ) class AuthUser(ABase): + + @hybrid_property + def id(self): + return self.user_id + @hybrid_property def password(self): return self.password_hash @@ -290,23 +304,10 @@ class AuthRole(ABase): def __unicode__(self, *args, **kwargs): return "Role %s" % (self.name,) -#class AuthUserRole(ABase): -# def __unicode__(self, *args, **kwargs): -# return "UserRole %s" % (self.id,) - class AuthRolePermission(ABase): def __unicode__(self, *args, **kwargs): return "%s" % (self.name,) -#@total_ordering -#class DeviceType(CUserType): -# device_id = columns.Text() -# device_os = columns.Text() -# push_id = columns.Text() -# app_version = columns.Text() -# registered = columns.DateTime(default=dt.datetime.utcnow) - - class User(ABase): """A user participates in the gamification, i.e. can get achievements, rewards, participate in leaderbaord etc.""" From d57e7dac93635d941f3171ae9661c0f198404f28 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 15 Jul 2016 09:38:44 +0200 Subject: [PATCH 026/176] minor authentication adjustments --- gengine/app/admin.py | 5 ++++- gengine/app/static/admin_layout.css | 12 +++++++++++- gengine/app/templates/admin/layout.html | 4 ++++ gengine/app/views.py | 2 +- gengine/maintenance/scripts/initializedb.py | 10 ++++++++-- 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/gengine/app/admin.py b/gengine/app/admin.py index 46eef0d..3bebc72 100644 --- a/gengine/app/admin.py +++ b/gengine/app/admin.py @@ -10,6 +10,7 @@ from flask_admin.contrib.sqla.filters import IntEqualFilter from flask_admin.contrib.sqla.view import ModelView from flask_admin.model.form import InlineFormAdmin +from pyramid.settings import asbool from wtforms import BooleanField from wtforms.form import Form @@ -17,6 +18,7 @@ GoalGoalProperty, Reward, User, GoalEvaluationCache, Value, AchievementUser, TranslationVariable, Language, Translation, \ AuthUser, AuthRole, AuthRolePermission from gengine.app.permissions import yield_all_perms +from gengine.base.settings import get_settings adminapp=None admin=None @@ -66,7 +68,8 @@ def init_admin(urlprefix="",secret="fKY7kJ2xSrbPC5yieEjV",override_admin=None,ov @adminapp.context_processor def inject_version(): - return { "gamification_engine_version" : pkg_resources.get_distribution("gamification-engine").version } + return { "gamification_engine_version" : pkg_resources.get_distribution("gamification-engine").version, + "settings_enable_authentication" : asbool(get_settings().get("enable_user_authentication",False))} if not override_admin: admin = Admin(adminapp, diff --git a/gengine/app/static/admin_layout.css b/gengine/app/static/admin_layout.css index ac2c36f..325b7cc 100644 --- a/gengine/app/static/admin_layout.css +++ b/gengine/app/static/admin_layout.css @@ -8,4 +8,14 @@ pre {outline: 1px solid #ccc; padding: 5px; margin: 5px; } .number { color: darkorange; } .boolean { color: blue; } .null { color: magenta; } -.key { color: red; } \ No newline at end of file +.key { color: red; } + +.auth_active { + color:green; + font-weight:bold; +} + +.auth_inactive { + color:red; + font-weight:bold; +} \ No newline at end of file diff --git a/gengine/app/templates/admin/layout.html b/gengine/app/templates/admin/layout.html index 12e3f9a..be1fd6c 100644 --- a/gengine/app/templates/admin/layout.html +++ b/gengine/app/templates/admin/layout.html @@ -61,6 +61,10 @@ {% endif %} {% endfor %} +
  • +
  • + Authentication: {{ ('active' if settings_enable_authentication else 'inactive') | safe }} +
    • {% endmacro %} {% macro messages() %} diff --git a/gengine/app/views.py b/gengine/app/views.py index f8e0655..0acb6ea 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -366,7 +366,7 @@ def _get_basicauth_credentials(request): if user: j = t_auth_users.join(t_auth_users_roles).join(t_auth_roles).join(t_auth_roles_permissions) - q = select([t_auth_roles_permissions.c.name], from_obj=j).where(t_auth_users.c.id==user.id) + q = select([t_auth_roles_permissions.c.name], from_obj=j).where(t_auth_users.c.user_id==user.user_id) permissions = [r["name"] for r in DBSession.execute(q).fetchall()] if not perm_global_access_admin_ui in permissions: return request_auth() diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index 53d9e6a..8cc886e 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -13,7 +13,8 @@ from sqlalchemy import engine_from_config from sqlalchemy.sql.schema import Table -from gengine.app.permissions import perm_global_delete_user, perm_global_increase_value, perm_global_update_user_infos +from gengine.app.permissions import perm_global_delete_user, perm_global_increase_value, perm_global_update_user_infos, \ + perm_global_access_admin_ui def usage(argv): @@ -245,17 +246,22 @@ def add_translation(variable, lang, text): DBSession.add(user1) DBSession.add(user2) DBSession.add(user3) + DBSession.flush() - auth_user = AuthUser(id=1,email="admin@gamification-software.com",password="test123",active=True) + auth_user = AuthUser(user_id=user1.id,email="admin@gamification-software.com",password="test123",active=True) DBSession.add(auth_user) auth_role = AuthRole(name="Global Admin") DBSession.add(auth_role) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_access_admin_ui)) DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_delete_user)) DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_increase_value)) DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_update_user_infos)) + auth_user.roles.append(auth_role) + DBSession.add(auth_user) + if __name__ == '__main__': main() \ No newline at end of file From 64a6b6553414369145e85c74fb25d78e9de2718e Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 18 Jul 2016 15:06:53 +0200 Subject: [PATCH 027/176] prepare pushes --- gengine/app/model.py | 100 ++++++++++++++--- gengine/app/permissions.py | 12 +++ gengine/app/push.py | 216 +++++++++++++++++++++++++++++++++++++ gengine/app/route.py | 4 +- gengine/app/views.py | 121 +++++++++++++++++++-- 5 files changed, 428 insertions(+), 25 deletions(-) create mode 100644 gengine/app/push.py diff --git a/gengine/app/model.py b/gengine/app/model.py index cc670ed..4421b54 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -84,15 +84,6 @@ def get_default_token_valid_time(): UniqueConstraint("role_id", "name") ) -t_user_device = Table('user_devices', Base.metadata, - Column('id', ty.BigInteger, primary_key = True), - Column('user_id', ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), primary_key = True, nullable=False), - Column('device_os', ty.String, nullable=False), - Column('push_id', ty.String(), nullable=False), - Column('app_version', ty.String(), nullable=False), - Column('registered_at', ty.DateTime(), nullable=False, default=datetime.datetime.utcnow), -) - t_users_users = Table("users_users", Base.metadata, Column('from_id', ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), primary_key = True, nullable=False), Column('to_id', ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), primary_key = True, nullable=False) @@ -243,6 +234,33 @@ def get_default_token_valid_time(): Column('text', ty.Text(), nullable = False), ) +t_user_device = Table('user_devices', Base.metadata, + Column('device_id', ty.String(255), primary_key = True), + Column('user_id', ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), primary_key = True, nullable=False), + Column('device_os', ty.String, nullable=False), + Column('push_id', ty.String(255), nullable=False), + Column('app_version', ty.String(255), nullable=False), + Column('registered_at', ty.DateTime(), nullable=False, default=datetime.datetime.utcnow), +) + +t_user_messages = Table('user_messages', Base.metadata, + Column('id', ty.BigInteger, primary_key = True), + Column('user_id', ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), index = True, nullable=False), + Column('translation_id', ty.Integer, ForeignKey("translationvariables.id", ondelete="RESTRICT"), nullable = True), + Column('params', ty.JSON, nullable=True, default={}), + Column('is_read', ty.Boolean, index=True, default=False, nullable=False), + Column('created_at', ty.DateTime(), nullable=False, default=datetime.datetime.utcnow, index=True), +) + +t_goal_triggers = Table('goal_triggers', Base.metadata, + Column('id', ty.Integer, primary_key = True), + Column('goal_id', ty.Integer, ForeignKey("goals.id", ondelete="CASCADE"), nullable=False, index=True), + Column('condition_type', ty.Enum("percentage", name="goal_trigger_condition_types"), default="percentage"), + Column('condition_percentage', ty.Float, nullable=True), + Column('action_type', ty.Enum("user_message", name="goal_trigger_action_types"), default="user_message"), + Column('action_translation_id', ty.Integer, ForeignKey("translationvariables.id", ondelete="RESTRICT"), nullable = True), +) + class AuthUser(ABase): @hybrid_property @@ -285,7 +303,6 @@ def get_or_create_token(self): return tokenObj - class AuthToken(ABase): @staticmethod @@ -308,6 +325,42 @@ class AuthRolePermission(ABase): def __unicode__(self, *args, **kwargs): return "%s" % (self.name,) +class UserDevice(ABase): + def __unicode__(self, *args, **kwargs): + return "Device: %s" % (self.id,) + + @classmethod + def add_or_update_device(cls, user_id, device_id, push_id, device_os, app_version): + device = DBSession.execute(t_user_device.select().where(and_( + t_user_device.c.device_id == device_id, + t_user_device.c.user_id == user_id + ))).fetchone() + + if device and (device["push_id"] != push_id + or device["device_os"] != device_os + or device["app_version"] != app_version + ): + uSession = update_connection() + q = t_user_device.update().values({ + "push_id": push_id, + "device_os": device_os, + "app_version": app_version + }).where(and_( + t_user_device.c.device_id == device_id, + t_user_device.c.user_id == user_id + )) + uSession.execute(q) + elif not device: # insert + uSession = update_connection() + q = t_user_device.insert().values({ + "push_id": push_id, + "device_os": device_os, + "app_version": app_version, + "device_id": device_id, + "user_id": user_id + }) + uSession.execute(q) + class User(ABase): """A user participates in the gamification, i.e. can get achievements, rewards, participate in leaderbaord etc.""" @@ -1276,6 +1329,17 @@ def get_translation_variable(cls,translation_id): def get_languages(cls): return DBSession.execute(t_languages.select()).fetchall() +class UserMessage(ABase): + def __unicode__(self, *args, **kwargs): + return "Message: %s" % (Translation.trs(self.translationvariable_id,self.params).get(_fallback_language),) + + def get_text(self, row): + return Translation.trs(row["translationvariable_id"],row["params"]) + +class GoalTrigger(ABase): + def __unicode__(self, *args, **kwargs): + return "GoalTrigger: %s" % (self.id,) + mapper(AuthUser, t_auth_users, properties={ 'roles' : relationship(AuthRole, secondary=t_auth_users_roles, backref="users") }) @@ -1284,11 +1348,6 @@ def get_languages(cls): 'user' : relationship(AuthUser, backref="tokens") }) -#mapper(AuthUserRole, t_auth_users_roles, properties={ -# 'user' : relationship(AuthUser, backref="roles"), -# 'role' : relationship(AuthRole, backref="users") -#}) - mapper(AuthRole, t_auth_roles, properties={ }) @@ -1303,6 +1362,10 @@ def get_languages(cls): secondaryjoin=t_users.c.id==t_users_users.c.to_id) }) +mapper(UserDevice, t_user_device, properties={ + 'user' : relationship(User, backref="devices"), +}) + mapper(Group, t_groups, properties={ 'users' : relationship(User, secondary=t_users_groups, backref="groups"), }) @@ -1356,6 +1419,11 @@ def get_languages(cls): 'goal' : relationship(Goal) }) +mapper(GoalTrigger,t_goal_triggers, properties={ + 'goal' : relationship(Goal,backref="triggers"), + 'value_translation' : relationship(TranslationVariable) +}) + mapper(Language, t_languages) mapper(TranslationVariable,t_translationvariables) mapper(Translation, t_translations, properties={ @@ -1372,5 +1440,3 @@ def insert_variable_for_property(mapper,connection,target): variable.name = target.name variable.group = "day" DBSession.add(variable) - - diff --git a/gengine/app/permissions.py b/gengine/app/permissions.py index adfd5c5..e7b1771 100644 --- a/gengine/app/permissions.py +++ b/gengine/app/permissions.py @@ -17,6 +17,18 @@ perm_global_increase_value = "global_increase_value" desc_global_increase_value = "(Admin) Increase every user's values" +perm_global_register_device = "global_register_device" +desc_global_register_device = "(Admin) Register devices for any user" + +perm_own_register_device = "own_register_device" +desc_own_register_device = "Register devices for myself" + +perm_global_read_messages= "global_read_messages" +desc_global_read_messages = "(Admin) Read messages of all users" + +perm_own_read_messages = "perm_own_read_messages" +desc_own_read_messages = "Read own messages" + def yield_all_perms(): for k,v in globals().items(): if k.startswith("perm_"): diff --git a/gengine/app/push.py b/gengine/app/push.py new file mode 100644 index 0000000..3cfc28c --- /dev/null +++ b/gengine/app/push.py @@ -0,0 +1,216 @@ +import logging +import random +import threading +from apns import APNs, Payload +from gcm import GCM +import os +from sqlalchemy.sql.expression import and_, select +from sqlalchemy.sql.functions import func + +from gengine.app.model import t_user_device, t_user_messages +from gengine.base.model import update_connection +from gengine.base.settings import get_settings +from gengine.metadata import DBSession + +threadlocal = threading.local() +log = logging.getLogger(__name__) + +def get_prod_apns(): + """ + http://stackoverflow.com/questions/1762555/creating-pem-file-for-apns + + Step 1: Create Certificate .pem from Certificate .p12 + Command: openssl pkcs12 -clcerts -nokeys -out apns-dev-cert.pem -in apns-dev-cert.p12 + + Step 2: Create Key .pem from Key .p12 + Command : openssl pkcs12 -nocerts -out apns-dev-key.pem -in apns-dev-key.p12 + + Step 3: If you want to remove pass phrase asked in second step + Command : openssl rsa -in apns-dev-key.pem -out apns-dev-key-noenc.pem + + """ + if not hasattr(threadlocal, "prod_apns"): + settings = get_settings() + cert_file = os.environ.get("APNS_CERT", settings.get("apns.prod.certificate")) + key_file = os.environ.get("APNS_KEY", settings.get("apns.prod.key")) + sandbox = False # other_helpers.boolify(os.environ.get("APNS_SANDBOX",settings.get("apns.sandbox"))) + threadlocal.prod_apns = APNs(use_sandbox=sandbox, cert_file=cert_file, key_file=key_file, enhanced=True) + + def response_listener(error_response): + log.debug("client get error-response: " + str(error_response)) + + threadlocal.prod_apns.gateway_server.register_response_listener(response_listener) + return threadlocal.prod_apns + +def get_dev_apns(): + """ + http://stackoverflow.com/questions/1762555/creating-pem-file-for-apns + + Step 1: Create Certificate .pem from Certificate .p12 + Command: openssl pkcs12 -clcerts -nokeys -out apns-dev-cert.pem -in apns-dev-cert.p12 + + Step 2: Create Key .pem from Key .p12 + Command : openssl pkcs12 -nocerts -out apns-dev-key.pem -in apns-dev-key.p12 + + Step 3: If you want to remove pass phrase asked in second step + Command : openssl rsa -in apns-dev-key.pem -out apns-dev-key-noenc.pem + + """ + if not hasattr(threadlocal, "dev_apns"): + settings = get_settings() + cert_file = os.environ.get("APNS_CERT", settings.get("apns.dev.certificate")) + key_file = os.environ.get("APNS_KEY", settings.get("apns.dev.key")) + sandbox = True # other_helpers.boolify(os.environ.get("APNS_SANDBOX",settings.get("apns.sandbox"))) + threadlocal.dev_apns = APNs(use_sandbox=sandbox, cert_file=cert_file, key_file=key_file, enhanced=True) + + def response_listener(error_response): + log.debug("client get error-response: " + str(error_response)) + + threadlocal.dev_apns.gateway_server.register_response_listener(response_listener) + return threadlocal.dev_apns + +def get_gcm(): + if not hasattr(threadlocal, "gcm"): + settings = get_settings() + # JSON request + API_KEY = os.environ.get("GCM_API_KEY", settings.get("gcm.api_key")) + threadlocal.gcm = GCM(API_KEY) + return threadlocal.gcm + +def prod_apns_feedback(): + apns_feedback(get_prod_apns(), "prod_") + +def dev_apns_feedback(): + apns_feedback(get_dev_apns(), "dev_") + +def apns_feedback(apns, prefix): + # Get feedback messages. + uS = update_connection() + + for (token_hex, fail_time) in apns.feedback_server.items(): + try: + if not isinstance(token_hex, str): + token_hex = token_hex.decode("utf8") + + token_hex = prefix + token_hex + + log.debug("APNS Feedback Entry: %s", token_hex + "_" + str(fail_time)) + + # do stuff with token_hex and fail_time + q = t_user_device.select().where(t_user_device.c.push_id==token_hex) + rows = uS.execute(q).fetchall() + + for device in rows: + log.debug("APNSPushID found in Database: %s", token_hex) + if fail_time > device["registered"]: + log.debug("Fail-Time is before Registered-At") + uS.execute(t_user_device.delete().where( + t_user_device.c.device_id == device["device_id"], + t_user_device.c.user_id == device["user_id"], + )) + except: + log.exception("Processing APNS Feedback failed for an entry.") + + +def gcm_feedback(response): + # Successfully handled registration_ids + if 'success' in response: + for reg_id, success_id in response['success'].items(): + log.debug('Successfully sent notification for reg_id {0}'.format(reg_id)) + + uS = update_connection() + + # Handling errors + if 'errors' in response: + + for error, reg_ids in response['errors'].items(): + # Check for errors and act accordingly + if error in ['NotRegistered', 'InvalidRegistration']: + # Remove reg_ids from database + for reg_id in reg_ids: + + q = t_user_device.select().where(t_user_device.c.push_id == reg_id) + rows = uS.execute(q).fetchall() + + for device in rows: + if not isinstance(reg_id, str): + reg_id = reg_id.decode("utf8") + + log.debug("Removing reg_id: {0} from db".format(reg_id)) + + uS.execute(t_user_device.delete().where( + t_user_device.c.device_id == device["device_id"], + t_user_device.c.user_id == device["user_id"], + )) + + # Repace reg_id with canonical_id in your database + if 'canonical' in response: + for reg_id, canonical_id in response['canonical'].items(): + if not isinstance(reg_id, str): + reg_id = reg_id.decode("utf8") + + log.debug("Replacing reg_id: {0} with canonical_id: {1} in db".format(reg_id, canonical_id)) + + q = t_user_device.update().values({ + "push_id" : canonical_id + }).where(and_( + t_user_device.c.push_id == reg_id, + )) + + uS.execute(q) + +def send_push_message( + user_id, + text="", + custom_payload={}, + title="Gamification-Engine", + android_text=None, + ios_text=None): + + message_count = DBSession.execute(select([func.count("*").label("c")],from_obj=t_user_messages).where(and_( + t_user_messages.c.user_id == user_id, + t_user_messages.c.is_read == False + ))).scalar() + + data = dict({"title": title, + "badge": message_count}, **custom_payload) + + settings = get_settings() + + if not ios_text: + ios_text = text + + if not android_text: + android_text = text + + rows = DBSession.execute(t_user_device.select().where(t_user_device.c.user_id==user_id)).fetchall() + + for device in rows: + + if "ios" in device.device_os.lower(): + identifier = random.getrandbits(32) + + if custom_payload: + payload = Payload(alert=ios_text, custom=data, badge=message_count, sound="default") + else: + payload = Payload(alert=ios_text, custom=data, badge=message_count, sound="default") + + log.debug("Sending Push message to User (ID: %s): %s", user_id, ios_text) + + if device.push_id.startswith("prod_"): + get_prod_apns().gateway_server.send_notification(device.push_id[5:], payload, identifier=identifier) + elif device.push_id.startswith("dev_"): + get_dev_apns().gateway_server.send_notification(device.push_id[4:], payload, identifier=identifier) + + if "android" in device.device_os.lower(): + + log.debug("Sending Push message to User (ID: %s): %s", user_id, android_text) + push_id = device.push_id.lstrip("dev_").lstrip("prod_") + + response = get_gcm().json_request(registration_ids=[push_id, ], + data={"message": android_text, "data": data, "title": title}, + restricted_package_name=os.environ.get("GCM_PACKAGE", settings.get("gcm.package","")), + priority='high', + delay_while_idle=False) + if response: + gcm_feedback(response) \ No newline at end of file diff --git a/gengine/app/route.py b/gengine/app/route.py index 1b2bcf2..74188db 100644 --- a/gengine/app/route.py +++ b/gengine/app/route.py @@ -8,4 +8,6 @@ def config_routes(config): config.add_route('delete_user', '/delete_user/{user_id}') config.add_route('get_achievement_level', '/achievement/{achievement_id}/level/{level}') - config.add_route('auth_login', '/auth/login') \ No newline at end of file + config.add_route('auth_login', '/auth/login') + + config.add_route('register_device', '/register_device/{user_id}') \ No newline at end of file diff --git a/gengine/app/views.py b/gengine/app/views.py index 0acb6ea..9aa11fc 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -10,11 +10,12 @@ from pyramid.request import Request from pyramid.response import Response from pyramid.settings import asbool -from sqlalchemy.sql.expression import select +from sqlalchemy.sql.expression import select, and_ from gengine.app.permissions import perm_own_update_user_infos, perm_global_update_user_infos, perm_global_delete_user, perm_own_delete_user, \ - perm_global_access_admin_ui -from gengine.base.model import valid_timezone, exists_by_expr + perm_global_access_admin_ui, perm_global_register_device, perm_own_register_device, perm_global_read_messages, \ + perm_own_read_messages +from gengine.base.model import valid_timezone, exists_by_expr, update_connection from gengine.base.errors import APIError from pyramid.exceptions import NotFound from pyramid.renderers import render @@ -29,7 +30,8 @@ Achievement, Value, Variable, - AuthUser, AuthToken, t_users, t_auth_users, t_auth_users_roles, t_auth_roles, t_auth_roles_permissions) + AuthUser, AuthToken, t_users, t_auth_users, t_auth_users_roles, t_auth_roles, t_auth_roles_permissions, UserDevice, + t_user_device, t_user_messages, UserMessage) from gengine.base.settings import get_settings from gengine.metadata import DBSession from gengine.wsgiutil import HTTPSProxied @@ -288,15 +290,15 @@ def auth_login(request): password = doc.get("password") if not email or not password: - raise APIError(404, "login.email_and_password_required", "You need to send your email and password.") + raise APIError(400, "login.email_and_password_required", "You need to send your email and password.") user = DBSession.query(AuthUser).filter_by(email=email).first() if not user or not user.verify_password(password): - raise APIError(404, "login.email_or_password_invalid", "Either the email address or the password is wrong.") + raise APIError(401, "login.email_or_password_invalid", "Either the email address or the password is wrong.") if not user.active: - raise APIError(404, "user_is_not_activated", "Your user is not activated.") + raise APIError(400, "user_is_not_activated", "Your user is not activated.") token = AuthToken.generate_token() tokenObj = AuthToken( @@ -311,6 +313,111 @@ def auth_login(request): "token" : token } +@view_config(route_name='register_device', renderer='json', request_method="POST") +def register_device(request): + try: + doc = request.json_body + except: + raise APIError(400, "invalid_json", "no valid json body") + + user_id = int(request.matchdict["user_id"]) + + device_id = doc.get("device_id") + push_id = doc.get("push_id") + device_os = doc.get("device_os") + app_version = doc.get("app_version") + + if not device_id \ + or not push_id \ + or not user_id \ + or not device_os \ + or not app_version: + raise APIError(400, "register_device.required_fields", + "Required fields: device_id, push_id, device_os, app_version") + + if asbool(get_settings().get("enable_user_authentication", False)): + may_register = request.has_perm(perm_global_register_device) or request.has_perm( + perm_own_register_device) and str(request.user.id) == str(user_id) + if not may_register: + raise APIError(403, "forbidden", "You may not register devices for this user.") + + if not exists_by_expr(t_users, t_users.c.id==user_id): + raise APIError(404, "register_device.user_not_found", + "There is no user with this id.") + + UserDevice.add_or_update_device(user_id = user_id, device_id = device_id, push_id = push_id, device_os = device_os, app_version = app_version) + + return { + "status" : "ok" + } + +@view_config(route_name='get_messages', renderer='json', request_method="GET") +def get_messages(request): + user_id = int(request.matchdict["user_id"]) + offset = int(request.GET.get("offset",0)) + limit = 100 + + if asbool(get_settings().get("enable_user_authentication", False)): + may_read_messages = request.has_perm(perm_global_read_messages) or request.has_perm( + perm_own_read_messages) and str(request.user.id) == str(user_id) + if not may_read_messages: + raise APIError(403, "forbidden", "You may not read the messages of this user.") + + if not exists_by_expr(t_users, t_users.c.id == user_id): + raise APIError(404, "get_messages.user_not_found", + "There is no user with this id.") + + q = t_user_messages.select().where(t_user_messages.c.user_id==user_id).order_by(t_user_messages.c.created_at.desc()).limit(limit).offset(offset) + rows = DBSession.execute(q).fetchall() + + return { + "messages" : [{ + "id" : message["id"], + "text" : UserMessage.get_text(message), + "is_read" : message["is_read"], + "created_at" : message["created_at"] + } for message in rows] + } + + +@view_config(route_name='read_messages', renderer='json', request_method="POST") +def set_messages_read(request): + try: + doc = request.json_body + except: + raise APIError(400, "invalid_json", "no valid json body") + + user_id = int(request.matchdict["user_id"]) + + if asbool(get_settings().get("enable_user_authentication", False)): + may_read_messages = request.has_perm(perm_global_read_messages) or request.has_perm( + perm_own_read_messages) and str(request.user.id) == str(user_id) + if not may_read_messages: + raise APIError(403, "forbidden", "You may not read the messages of this user.") + + if not exists_by_expr(t_users, t_users.c.id == user_id): + raise APIError(404, "set_messages_read.user_not_found", "There is no user with this id.") + + message_id = doc.get("message_id") + q = select([t_user_messages.c.id, + t_user_messages.c.created_at], from_obj=message_id).where(and_(t_user_messages.c.id==message_id, + t_user_messages.c.user_id==user_id)) + msg = DBSession.execute(q).fetchone() + if not msg: + raise APIError(404, "set_messages_read.message_not_found", "There is no message with this id.") + + uS = update_connection() + uS.execute(t_user_messages.update().values({ + "is_read" : True + }).where(and_( + t_user_messages.c.user_id == user_id, + t_user_messages.c.created_at <= msg["created_at"] + ))) + + return { + "status" : "ok" + } + @view_config(route_name='admin_app') @wsgiapp2 def admin_tenant(environ, start_response): From d63e968e9d33596369b02bdab01c7175f19f7755 Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 18 Jul 2016 22:47:03 +0200 Subject: [PATCH 028/176] fix a small regex bug for the string templates --- gengine/app/formular.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/app/formular.py b/gengine/app/formular.py index b51fe9d..72a1ddf 100644 --- a/gengine/app/formular.py +++ b/gengine/app/formular.py @@ -133,7 +133,7 @@ def evaluate_value_expression(expression, params={}): def render_string(tpl, params): """Substitute text in <> with corresponding variable value.""" - regex = re.compile('\${(.+)}') + regex = re.compile('\${(.+?)}') def repl(m): group = m.group(1) value = evaluate_value_expression(group, params) From d531917d80f33bbab35ca6ec0a9c72d70f822387 Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 18 Jul 2016 22:47:36 +0200 Subject: [PATCH 029/176] implement triggers, messages... --- gengine/app/admin.py | 38 ++++++++++++- gengine/app/model.py | 127 +++++++++++++++++++++++++++++++++++++++--- gengine/app/route.py | 4 +- gengine/app/views.py | 38 +++++++------ gengine/base/model.py | 12 ++++ gengine/metadata.py | 10 +++- setup.py | 7 ++- 7 files changed, 204 insertions(+), 32 deletions(-) diff --git a/gengine/app/admin.py b/gengine/app/admin.py index 3bebc72..27aa3c2 100644 --- a/gengine/app/admin.py +++ b/gengine/app/admin.py @@ -16,7 +16,7 @@ from gengine.app.model import DBSession, Variable, Goal, AchievementCategory, Achievement, AchievementProperty, GoalProperty, AchievementAchievementProperty, AchievementReward,\ GoalGoalProperty, Reward, User, GoalEvaluationCache, Value, AchievementUser, TranslationVariable, Language, Translation, \ - AuthUser, AuthRole, AuthRolePermission + AuthUser, AuthRole, AuthRolePermission, GoalTrigger, GoalTriggerStep, UserMessage from gengine.app.permissions import yield_all_perms from gengine.base.settings import get_settings @@ -82,6 +82,8 @@ def inject_version(): admin.add_view(ModelViewAchievement(DBSession, category="Rules")) admin.add_view(ModelViewGoal(DBSession, category="Rules")) + admin.add_view(ModelViewGoalTrigger(DBSession, category="Rules")) + admin.add_view(ModelView(AchievementAchievementProperty, DBSession, category="Rules", name="Achievement Property Values")) admin.add_view(ModelView(AchievementReward, DBSession, category="Rules", name="Achievement Reward Values")) admin.add_view(ModelView(GoalGoalProperty, DBSession, category="Rules", name="Goal Property Values")) @@ -103,6 +105,7 @@ def inject_version(): admin.add_view(ModelViewGoalEvaluationCache(DBSession, category="Debug")) admin.add_view(ModelViewUser(DBSession, category="Debug")) admin.add_view(ModelView(AchievementUser, DBSession, category="Debug")) + admin.add_view(ModelViewUserMessage(DBSession, category="Debug")) class TranslationInlineModelForm(InlineFormAdmin): form_columns = ('id','language','text') @@ -139,9 +142,30 @@ class ModelViewVariable(ModelView): def __init__(self, session, **kwargs): super(ModelViewVariable, self).__init__(Variable, session, **kwargs) +class GoalTriggerStepInlineModelForm(InlineFormAdmin): + form_columns = ( + 'id', + 'step', + 'condition_type', + 'condition_percentage', + 'action_type', + 'action_translation', + ) + +class ModelViewGoalTrigger(ModelView): + form_columns = ( + 'name', + 'goal', + 'steps', + ) + inline_models = (GoalTriggerStepInlineModelForm(GoalTriggerStep),) + + def __init__(self, session, **kwargs): + super(ModelViewGoalTrigger, self).__init__(GoalTrigger, session, **kwargs) + class ModelViewGoal(ModelView): column_list = ('condition','evaluation','operator','goal','timespan','priority','achievement','updated_at') - form_excluded_columns =('properties',) + form_excluded_columns =('properties','triggers') #column_searchable_list = ('name',) column_filters = (Achievement.id,) fast_mass_delete = True @@ -248,3 +272,13 @@ class ModelViewAuthRole(ModelView): def __init__(self, session, **kwargs): super(ModelViewAuthRole, self).__init__(AuthRole, session, **kwargs) + + +class ModelViewUserMessage(ModelView): + column_list = ('user','text','created_at','is_read') + column_details_list = ('user', 'text', 'created_at', 'is_read', 'params') + can_edit = False + can_view_details = True + + def __init__(self, session, **kwargs): + super(ModelViewUserMessage, self).__init__(UserMessage, session, **kwargs) diff --git a/gengine/app/model.py b/gengine/app/model.py index 4421b54..d4f1703 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -2,14 +2,17 @@ """models including business logic""" import datetime +import logging from datetime import timedelta import hashlib import pytz import sqlalchemy.types as ty +import sys + from pyramid.settings import asbool from sqlalchemy.ext.hybrid import hybrid_property -from sqlalchemy.sql.schema import UniqueConstraint +from sqlalchemy.sql.schema import UniqueConstraint, Index from gengine.app.permissions import perm_global_increase_value from gengine.base.model import ABase, exists_by_expr, datetime_trunc, calc_distance, coords, update_connection @@ -37,6 +40,8 @@ from gengine.app.formular import evaluate_condition, evaluate_value_expression, evaluate_string +log = logging.getLogger(__name__) + t_users = Table("users", Base.metadata, Column('id', ty.BigInteger, primary_key = True), Column("lat", ty.Float(Precision=64), nullable=True), @@ -254,11 +259,29 @@ def get_default_token_valid_time(): t_goal_triggers = Table('goal_triggers', Base.metadata, Column('id', ty.Integer, primary_key = True), + Column("name", ty.String(100), nullable=False), Column('goal_id', ty.Integer, ForeignKey("goals.id", ondelete="CASCADE"), nullable=False, index=True), +) + +t_goal_trigger_steps = Table('goal_trigger_steps', Base.metadata, + Column('id', ty.Integer, primary_key = True), + Column('goal_trigger_id', ty.Integer, ForeignKey("goal_triggers.id", ondelete="CASCADE"), nullable=False, index=True), + Column('step', ty.Integer, nullable=False, default=0), Column('condition_type', ty.Enum("percentage", name="goal_trigger_condition_types"), default="percentage"), Column('condition_percentage', ty.Float, nullable=True), Column('action_type', ty.Enum("user_message", name="goal_trigger_action_types"), default="user_message"), Column('action_translation_id', ty.Integer, ForeignKey("translationvariables.id", ondelete="RESTRICT"), nullable = True), + + UniqueConstraint("goal_trigger_id", "step") +) + +t_goal_trigger_step_executions = Table('goal_trigger_executions', Base.metadata, + Column('id', ty.BigInteger, primary_key = True), + Column('trigger_step_id', ty.Integer, ForeignKey("goal_trigger_steps.id", ondelete="CASCADE"), nullable=False), + Column('user_id', ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), nullable=False), + Column('execution_level', ty.Integer, nullable = False, default=0), + Column('execution_date', ty.DateTime(), nullable=False, default=datetime.datetime.utcnow, index=True), + Index("ix_goal_trigger_executions_combined", "trigger_step_id","user_id","execution_level") ) class AuthUser(ABase): @@ -761,7 +784,7 @@ def basic_output(cls,achievement,goals,include_levels=True, "value" : evaluate_string(r["value"], {"level":i}), "value_translated" : Translation.trs(r["value_translation_id"], {"level":i}), } for r in Achievement.get_achievement_properties(achievement["id"],i)} - } for i in range(0,max_level_included+1)} + } for i in range(1,max_level_included+1)} return out @classmethod @@ -993,7 +1016,7 @@ class Goal(ABase): """A Goal defines a rule on variables that needs to be reached to get achievements""" def __unicode__(self, *args, **kwargs): - if self.name_translation!=None: + if self.name_translation_id!=None: name = Translation.trs(self.name_translation.id, {"level":1, "goal":'0'})[_fallback_language] return str(name) + " (ID: %s)" % (self.id,) else: @@ -1134,6 +1157,9 @@ def evaluate(cls, goal, user_id, level, goal_eval_cache_before=False): goal_achieved = True new = max(new,goal_goal) + # Evaluate triggers + Goal.select_and_execute_triggers(goal = goal, user_id = user_id, level = level, goal_goal = goal_goal, value = new) + return Goal.set_goal_eval_cache(goal=goal, user_id=user_id, value=new, @@ -1141,6 +1167,49 @@ def evaluate(cls, goal, user_id, level, goal_eval_cache_before=False): else: return Goal.get_goal_eval_cache(goal["id"], user_id) + @classmethod + def select_and_execute_triggers(cls, goal, user_id, level, goal_goal, value): + j = t_goal_trigger_step_executions.join(t_goal_trigger_steps) + executions = {r["goal_trigger_id"] : r["step"] for r in + DBSession.execute( + select([t_goal_trigger_steps.c.id.label("step_id"), + t_goal_trigger_steps.c.goal_trigger_id, + t_goal_trigger_steps.c.step], from_obj=j).\ + where(and_(t_goal_triggers.c.goal_id == goal["id"], + t_goal_trigger_step_executions.c.user_id == user_id, + t_goal_trigger_step_executions.c.execution_level == level))).fetchall() + } + + j = t_goal_trigger_steps.join(t_goal_triggers) + trigger_steps = DBSession.execute(t_goal_trigger_steps.select(from_obj=j).\ + where(t_goal_triggers.c.goal_id == goal["id"],)).\ + fetchall() + trigger_steps = [s for s in trigger_steps if s["step"]>executions.get(s["goal_trigger_id"],-sys.maxsize)] + if len(trigger_steps)>0: + operator = goal["operator"] + + goal_properties = { + p["name"]: (p["value_translated"] if p["value_translated"] else p["value"]) + for p in Goal.basic_goal_output(goal, level).get("properties").values() + } + + for step in trigger_steps: + if step["condition_type"] == "percentage" and step["condition_percentage"]: + current_percentage = float(value) / float(goal_goal) + required_percentage = step["condition_percentage"] + if (operator == "geq" and current_percentage >= required_percentage) \ + or (operator == "leq" and current_percentage <= required_percentage): + GoalTriggerStep.execute( + trigger_step = step, + user_id = user_id, + current_percentage = current_percentage, + value = value, + goal_goal = goal_goal, + goal_level = level, + goal_properties = goal_properties + ) + + @classmethod def get_goal_eval_cache(cls,goal_id,user_id): """lookup and return cache entry, else return None""" @@ -1303,9 +1372,12 @@ def trs(cls,translation_id,params={}): if translation_id is None: return None try: + # TODO support params which are results of this function itself (dicts of lang -> value) + # maybe even better: add possibility to refer to other translationvariables directly (so they can be modified later on) ret = {str(x["name"]) : evaluate_string(x["text"],params) for x in cls.get_translation_variable(translation_id)} - except: + except Exception as e: ret = {str(x["name"]) : x["text"] for x in cls.get_translation_variable(translation_id)} + log.exception("Evaluation of string-forumlar failed: %s" % (ret.get(_fallback_language,translation_id),)) if not _fallback_language in ret: ret[_fallback_language] = "[not_translated]_"+str(translation_id) @@ -1331,15 +1403,46 @@ def get_languages(cls): class UserMessage(ABase): def __unicode__(self, *args, **kwargs): - return "Message: %s" % (Translation.trs(self.translationvariable_id,self.params).get(_fallback_language),) + return "Message: %s" % (Translation.trs(self.translation_id,self.params).get(_fallback_language),) + + @classmethod + def get_text(cls, row): + return Translation.trs(row["translation_id"],row["params"]) - def get_text(self, row): - return Translation.trs(row["translationvariable_id"],row["params"]) + @property + def text(self): + return Translation.trs(self.translation_id, self.params) class GoalTrigger(ABase): def __unicode__(self, *args, **kwargs): return "GoalTrigger: %s" % (self.id,) +class GoalTriggerStep(ABase): + def __unicode__(self, *args, **kwargs): + return "GoalTriggerStep: %s" % (self.id,) + + @classmethod + def execute(cls, trigger_step, user_id, current_percentage, value, goal_goal, goal_level, goal_properties): + uS = update_connection() + uS.execute(t_goal_trigger_step_executions.insert().values({ + 'user_id' : user_id, + 'trigger_step_id' : trigger_step["id"], + 'execution_level' : goal_level, + })) + if trigger_step["action_type"] == "user_message": + m = UserMessage( + user_id = user_id, + translation_id = trigger_step["action_translation_id"], + params = dict({ + 'value' : value, + 'goal' : goal_goal, + 'percentage' : current_percentage + },**goal_properties), + is_read = False, + ) + uS.add(m) + + mapper(AuthUser, t_auth_users, properties={ 'roles' : relationship(AuthRole, secondary=t_auth_users_roles, backref="users") }) @@ -1421,7 +1524,10 @@ def __unicode__(self, *args, **kwargs): mapper(GoalTrigger,t_goal_triggers, properties={ 'goal' : relationship(Goal,backref="triggers"), - 'value_translation' : relationship(TranslationVariable) +}) +mapper(GoalTriggerStep,t_goal_trigger_steps, properties={ + 'trigger' : relationship(GoalTrigger,backref="steps"), + 'action_translation' : relationship(TranslationVariable) }) mapper(Language, t_languages) @@ -1431,6 +1537,11 @@ def __unicode__(self, *args, **kwargs): 'translationvariable' : relationship(TranslationVariable, backref="translations"), }) +mapper(UserMessage, t_user_messages, properties = { + 'user' : relationship(User, backref="user_messages"), + 'translationvariable' : relationship(TranslationVariable), +}) + @event.listens_for(AchievementProperty, "after_insert") @event.listens_for(AchievementProperty, 'after_update') def insert_variable_for_property(mapper,connection,target): diff --git a/gengine/app/route.py b/gengine/app/route.py index 74188db..05ffa55 100644 --- a/gengine/app/route.py +++ b/gengine/app/route.py @@ -10,4 +10,6 @@ def config_routes(config): config.add_route('auth_login', '/auth/login') - config.add_route('register_device', '/register_device/{user_id}') \ No newline at end of file + config.add_route('register_device', '/register_device/{user_id}') + config.add_route('get_messages', '/messages/{user_id}') + config.add_route('read_messages', '/read_messages/{user_id}') \ No newline at end of file diff --git a/gengine/app/views.py b/gengine/app/views.py index 9aa11fc..11ed6f8 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -152,19 +152,16 @@ def may_view(achievement, requesting_user): return ret -@view_config(route_name='get_progress', renderer='string') +@view_config(route_name='get_progress', renderer='json', request_method="GET") def get_progress(request): """get all relevant data concerning the user's progress""" user_id = int(request.matchdict["user_id"]) user = User.get_user(user_id) if not user: - raise NotFound("user not found") + raise APIError(404, "user_not_found", "user not found") - request.response.content_type = "application/json" - progress = _get_progress(achievements_for_user=user, requesting_user=request.user) - json_string, pmap = progress - return json_string + return _get_progress(achievements_for_user=user, requesting_user=request.user) @view_config(route_name='increase_value', renderer='json', request_method="POST") @view_config(route_name='increase_value_with_key', renderer='json', request_method="POST") @@ -422,17 +419,18 @@ def set_messages_read(request): @wsgiapp2 def admin_tenant(environ, start_response): - def admin_app(): - return HTTPSProxied(DebuggedApplication(adminapp.wsgi_app, True))(environ, start_response) + def admin_app(environ, start_response): + #return HTTPSProxied(DebuggedApplication(adminapp.wsgi_app, True))(environ, start_response) + return HTTPSProxied(adminapp.wsgi_app)(environ, start_response) - def request_auth(): + def request_auth(environ, start_response): resp = Response() resp.status_code = 401 resp.www_authenticate = 'Basic realm="%s"' % ("Gamification Engine Admin",) return resp(environ, start_response) if not asbool(get_settings().get("enable_user_authentication", False)): - return admin_app() + return admin_app(environ, start_response) req = Request(environ) @@ -469,20 +467,24 @@ def _get_basicauth_credentials(request): if cred: user = DBSession.query(AuthUser).filter_by(email=cred["login"]).first() if not user or not user.verify_password(cred["password"]): - return request_auth() + return request_auth(environ, start_response) if user: j = t_auth_users.join(t_auth_users_roles).join(t_auth_roles).join(t_auth_roles_permissions) q = select([t_auth_roles_permissions.c.name], from_obj=j).where(t_auth_users.c.user_id==user.user_id) permissions = [r["name"] for r in DBSession.execute(q).fetchall()] if not perm_global_access_admin_ui in permissions: - return request_auth() + return request_auth(environ, start_response) else: - cookie = SimpleCookie() - cookie['X-Auth-Token'] = user.get_or_create_token().token - cookie['X-Auth-Token']['path'] = get_settings().get("urlprefix","").rstrip("/")+"/" - cookieheaders = ('Set-Cookie', cookie['X-Auth-Token'].OutputString()) - start_response(200,[cookieheaders,]) + def start_response_with_headers(status, headers, exc_info=None): + + cookie = SimpleCookie() + cookie['X-Auth-Token'] = user.get_or_create_token().token + cookie['X-Auth-Token']['path'] = get_settings().get("urlprefix", "").rstrip("/") + "/" + + headers.append(('Set-Cookie', cookie['X-Auth-Token'].OutputString()),) + + return start_response(status, headers, exc_info) - return admin_app() \ No newline at end of file + return admin_app(environ, start_response_with_headers) \ No newline at end of file diff --git a/gengine/base/model.py b/gengine/base/model.py index 31bb28a..87a3a22 100644 --- a/gengine/base/model.py +++ b/gengine/base/model.py @@ -1,6 +1,7 @@ import pytz from pytz.exceptions import UnknownTimeZoneError from sqlalchemy.inspection import inspect +from sqlalchemy.orm.exc import DetachedInstanceError from sqlalchemy.sql.expression import select from sqlalchemy.sql.functions import func from zope.sqlalchemy.datamanager import mark_changed @@ -12,6 +13,17 @@ class ABaseMeta(type): def __init__(cls, name, bases, nmspc): super(ABaseMeta, cls).__init__(name, bases, nmspc) + # monkey patch __unicode__ + # this is required to give show the SQL error to the user in flask admin if constraints are violated + if hasattr(cls,"__unicode__"): + old_unicode = cls.__unicode__ + def patched(self): + try: + return old_unicode(self) + except DetachedInstanceError: + return "(DetachedInstance)" + cls.__unicode__ = patched + def __getattr__(cls, item): if item == "__table__": return inspect(cls).local_table diff --git a/gengine/metadata.py b/gengine/metadata.py index 486e7f7..2a6d69f 100644 --- a/gengine/metadata.py +++ b/gengine/metadata.py @@ -9,7 +9,7 @@ class MySession(Session): """This allow us to use the flask-admin sqla extension, which uses DBSession.commit() rather than transaction.commit()""" def commit(self,*args,**kw): transaction.commit(*args,**kw) - + def rollback(self,*args,**kw): transaction.abort(*args,**kw) @@ -20,7 +20,13 @@ def init_session(override_session=None): if override_session: DBSession = override_session else: - DBSession = scoped_session(sessionmaker(extension=ZopeTransactionExtension(), class_=MySession)) + DBSession = scoped_session( + sessionmaker( + extension=ZopeTransactionExtension(), + class_=MySession, + expire_on_commit=False + ) + ) Base=None diff --git a/setup.py b/setup.py index a3874a4..349e055 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,9 @@ 'jsl', 'jsonschema', 'pyparsing', - 'argon2' + 'argon2', + 'apns', + 'python-gcm' ] version = '' @@ -75,4 +77,7 @@ [redgalaxy.plugins] gengine = gengine:redgalaxy """, + dependency_links=[ + "https://github.com/ActiDoo/PyAPNs/archive/master.zip#egg=apns", + ], ) From 80edb8b12c606b749e12cd8030d31c055efb774f Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 19 Jul 2016 16:38:16 +0200 Subject: [PATCH 030/176] script for alembic revision generation --- .../maintenance/scripts/generate_revision.py | 85 +++++++++++++++++++ setup.py | 1 + 2 files changed, 86 insertions(+) create mode 100644 gengine/maintenance/scripts/generate_revision.py diff --git a/gengine/maintenance/scripts/generate_revision.py b/gengine/maintenance/scripts/generate_revision.py new file mode 100644 index 0000000..7e27a41 --- /dev/null +++ b/gengine/maintenance/scripts/generate_revision.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +import sys + +import os +import pyramid_dogpile_cache + +from pyramid.config import Configurator +from pyramid.paster import ( + get_appsettings, + setup_logging, +) +from pyramid.scripts.common import parse_vars +from sqlalchemy import engine_from_config + +def usage(argv): + cmd = os.path.basename(argv[0]) + print('usage: %s [var=value]\n' + '(example: "%s production.ini new_table_xy_created")' % (cmd, cmd)) + sys.exit(1) + + +def main(argv=sys.argv): + if len(argv) < 3: + usage(argv) + config_uri = argv[1] + message = argv[2] + options = parse_vars(argv[3:]) + setup_logging(config_uri) + settings = get_appsettings(config_uri, options=options) + + durl = os.environ.get("DATABASE_URL") # heroku + if durl: + settings['sqlalchemy.url'] = durl + + murl = os.environ.get("MEMCACHED_URL") + if murl: + settings['urlcache_url'] = murl + + revision(settings, message, options) + + +def revision(settings, message, options): + engine = engine_from_config(settings, 'sqlalchemy.') + + config = Configurator(settings=settings) + pyramid_dogpile_cache.includeme(config) + + from gengine.metadata import ( + init_session, + init_declarative_base, + init_db + ) + init_session() + init_declarative_base() + init_db(engine) + + from gengine.metadata import ( + Base, + ) + + if options.get("reset_db", False): + Base.metadata.drop_all(engine) + engine.execute("DROP SCHEMA IF EXISTS public CASCADE") + + engine.execute("CREATE SCHEMA IF NOT EXISTS public") + + from alembic.config import Config + from alembic import command + + alembic_cfg = Config(attributes={ + 'engine': engine, + 'schema': 'public' + }) + script_location = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + 'app/alembic' + ) + alembic_cfg.set_main_option("script_location", script_location) + + command.revision(alembic_cfg,message,True) + + engine.dispose() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/setup.py b/setup.py index 349e055..5eda0ef 100644 --- a/setup.py +++ b/setup.py @@ -74,6 +74,7 @@ initialize_gengine_db = gengine.maintenance.scripts.initializedb:main gengine_quickstart = gengine.maintenance.scripts.quickstart:main generate_gengine_erd = gengine.maintenance.scripts.generate_erd:main + generate_gengine_revision = gengine.maintenance.scripts.generate_revision:main [redgalaxy.plugins] gengine = gengine:redgalaxy """, From 9a1bca10413bb1e2ec9702ee0c6149fdec4db8d8 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 19 Jul 2016 16:38:50 +0200 Subject: [PATCH 031/176] make orm objects dict-like --- gengine/base/model.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gengine/base/model.py b/gengine/base/model.py index 87a3a22..e55e53d 100644 --- a/gengine/base/model.py +++ b/gengine/base/model.py @@ -29,6 +29,7 @@ def __getattr__(cls, item): return inspect(cls).local_table raise AttributeError(item) + class ABase(object, metaclass=ABaseMeta): """abstract base class which introduces a nice constructor for the model classes.""" @@ -45,6 +46,13 @@ def __str__(self): if hasattr(self, "__unicode__"): return self.__unicode__() + def __getitem__(self, key): + return getattr(self,key) + + def __setitem__(self, key, item): + return setattr(self,key,item) + + def calc_distance(latlong1, latlong2): """generates a sqlalchemy expression for distance query in km From c0b6d0e09aa2f0c37ee84ddb4c31b1ee7f379faf Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 19 Jul 2016 16:39:23 +0200 Subject: [PATCH 032/176] work on triggers... bugfixes.... --- gengine/app/admin.py | 5 +- gengine/app/alembic/env.py | 2 + gengine/app/model.py | 144 ++++++++++++++++++++++++++++--------- gengine/app/views.py | 4 +- 4 files changed, 117 insertions(+), 38 deletions(-) diff --git a/gengine/app/admin.py b/gengine/app/admin.py index 27aa3c2..f289c4e 100644 --- a/gengine/app/admin.py +++ b/gengine/app/admin.py @@ -157,6 +157,7 @@ class ModelViewGoalTrigger(ModelView): 'name', 'goal', 'steps', + 'execute_when_complete' ) inline_models = (GoalTriggerStepInlineModelForm(GoalTriggerStep),) @@ -252,8 +253,8 @@ def index(self): return self.render(template="admin_maintenance.html") class ModelViewAuthUser(ModelView): - column_list = ('id', 'email', 'active', 'created_at') - form_columns = ('email', 'password', 'active', 'roles') + column_list = ('user_id', 'email', 'active', 'created_at') + form_columns = ('user_id','email', 'password', 'active', 'roles') column_labels = {'password': 'Password'} def __init__(self, session, **kwargs): diff --git a/gengine/app/alembic/env.py b/gengine/app/alembic/env.py index 25332d4..42b6ed0 100644 --- a/gengine/app/alembic/env.py +++ b/gengine/app/alembic/env.py @@ -32,6 +32,8 @@ # target_metadata = None +from gengine.app.model import * + # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option") diff --git a/gengine/app/model.py b/gengine/app/model.py index d4f1703..1eade48 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -31,7 +31,8 @@ from sqlalchemy.dialects.postgresql import TIMESTAMP from sqlalchemy.orm import ( mapper, - relationship + relationship as sa_relationship, + backref as sa_backref ) from sqlalchemy.sql import bindparam @@ -261,6 +262,7 @@ def get_default_token_valid_time(): Column('id', ty.Integer, primary_key = True), Column("name", ty.String(100), nullable=False), Column('goal_id', ty.Integer, ForeignKey("goals.id", ondelete="CASCADE"), nullable=False, index=True), + Column('execute_when_complete', ty.Boolean, nullable=False, server_default='0', default=False), ) t_goal_trigger_steps = Table('goal_trigger_steps', Base.metadata, @@ -1124,7 +1126,7 @@ def generate_statement_cache(): return DBSession.execute(q, {'user_id' : user_id}) @classmethod - def evaluate(cls, goal, user_id, level, goal_eval_cache_before=False): + def evaluate(cls, goal, user_id, level, goal_eval_cache_before=False, execute_triggers=True): """evaluate the goal for the user_ids and the level""" operator = goal["operator"] @@ -1157,8 +1159,11 @@ def evaluate(cls, goal, user_id, level, goal_eval_cache_before=False): goal_achieved = True new = max(new,goal_goal) + previous_goal = Goal.basic_goal_output(goal, level-1).get("goal_goal",0) + # Evaluate triggers - Goal.select_and_execute_triggers(goal = goal, user_id = user_id, level = level, goal_goal = goal_goal, value = new) + if execute_triggers: + Goal.select_and_execute_triggers(goal = goal, user_id = user_id, level = level, current_goal = goal_goal, previous_goal = previous_goal, value = new) return Goal.set_goal_eval_cache(goal=goal, user_id=user_id, @@ -1168,7 +1173,7 @@ def evaluate(cls, goal, user_id, level, goal_eval_cache_before=False): return Goal.get_goal_eval_cache(goal["id"], user_id) @classmethod - def select_and_execute_triggers(cls, goal, user_id, level, goal_goal, value): + def select_and_execute_triggers(cls, goal, user_id, level, current_goal, value, previous_goal): j = t_goal_trigger_step_executions.join(t_goal_trigger_steps) executions = {r["goal_trigger_id"] : r["step"] for r in DBSession.execute( @@ -1181,33 +1186,52 @@ def select_and_execute_triggers(cls, goal, user_id, level, goal_goal, value): } j = t_goal_trigger_steps.join(t_goal_triggers) - trigger_steps = DBSession.execute(t_goal_trigger_steps.select(from_obj=j).\ - where(t_goal_triggers.c.goal_id == goal["id"],)).\ - fetchall() + + trigger_steps = DBSession.execute(select([ + t_goal_trigger_steps.c.id, + t_goal_trigger_steps.c.goal_trigger_id, + t_goal_trigger_steps.c.step, + t_goal_trigger_steps.c.condition_type, + t_goal_trigger_steps.c.condition_percentage, + t_goal_trigger_steps.c.action_type, + t_goal_trigger_steps.c.action_translation_id, + t_goal_triggers.c.execute_when_complete, + ],from_obj=j).\ + where(t_goal_triggers.c.goal_id == goal["id"],)).fetchall() + trigger_steps = [s for s in trigger_steps if s["step"]>executions.get(s["goal_trigger_id"],-sys.maxsize)] + + exec_queue = {} + + #When editing things here, check the insert_trigger_step_executions_after_step_upsert event listener too!!!!!!! if len(trigger_steps)>0: operator = goal["operator"] - goal_properties = { - p["name"]: (p["value_translated"] if p["value_translated"] else p["value"]) - for p in Goal.basic_goal_output(goal, level).get("properties").values() - } + goal_properties = Goal.get_properties(goal,level) for step in trigger_steps: if step["condition_type"] == "percentage" and step["condition_percentage"]: - current_percentage = float(value) / float(goal_goal) + current_percentage = float(value - previous_goal) / float(current_goal - previous_goal) required_percentage = step["condition_percentage"] + if current_percentage>=1.0 and required_percentage!=1.0 and not step["execute_when_complete"]: + # When the user reaches the full goal, and there is a trigger at e.g. 90%, we don't want it to be executed anymore. + continue if (operator == "geq" and current_percentage >= required_percentage) \ or (operator == "leq" and current_percentage <= required_percentage): - GoalTriggerStep.execute( - trigger_step = step, - user_id = user_id, - current_percentage = current_percentage, - value = value, - goal_goal = goal_goal, - goal_level = level, - goal_properties = goal_properties - ) + if exec_queue.get(step["goal_trigger_id"],{"step" : -sys.maxsize})["step"] < step["step"]: + exec_queue[step["goal_trigger_id"]] = step + + for step in exec_queue.values(): + current_percentage = float(value - previous_goal) / float(current_goal - previous_goal) + GoalTriggerStep.execute( + trigger_step = step, + user_id = user_id, + current_percentage = current_percentage, + value = value, + goal_goal = current_goal, + goal_level = level, + goal_properties = goal_properties + ) @classmethod @@ -1350,6 +1374,13 @@ def basic_goal_output(cls,goal,level): #"updated_at" : goal["updated_at"] } + @classmethod + @cache_general.cache_on_arguments() + def get_properties(cls, goal, level): + return { + p["name"]: (p["value_translated"] if p["value_translated"] else p["value"]) + for p in Goal.basic_goal_output(goal, level).get("properties").values() + } class Language(ABase): def __unicode__(self, *args, **kwargs): @@ -1422,26 +1453,71 @@ def __unicode__(self, *args, **kwargs): return "GoalTriggerStep: %s" % (self.id,) @classmethod - def execute(cls, trigger_step, user_id, current_percentage, value, goal_goal, goal_level, goal_properties): + def execute(cls, trigger_step, user_id, current_percentage, value, goal_goal, goal_level, goal_properties, suppress_actions=False): uS = update_connection() uS.execute(t_goal_trigger_step_executions.insert().values({ 'user_id' : user_id, 'trigger_step_id' : trigger_step["id"], 'execution_level' : goal_level, })) - if trigger_step["action_type"] == "user_message": - m = UserMessage( - user_id = user_id, - translation_id = trigger_step["action_translation_id"], - params = dict({ - 'value' : value, - 'goal' : goal_goal, - 'percentage' : current_percentage - },**goal_properties), - is_read = False, + if not suppress_actions: + if trigger_step["action_type"] == "user_message": + m = UserMessage( + user_id = user_id, + translation_id = trigger_step["action_translation_id"], + params = dict({ + 'value' : value, + 'goal' : goal_goal, + 'percentage' : current_percentage + },**goal_properties), + is_read = False, + ) + uS.add(m) + +@event.listens_for(GoalTriggerStep, "after_insert") +@event.listens_for(GoalTriggerStep, 'after_update') +def insert_trigger_step_executions_after_step_upsert(mapper,connection,target): + """When we create a new Trigger-Step, we must ensure, that is will not be executed for the users who already met the conditions before.""" + + user_ids = [x["id"] for x in DBSession.execute(select([t_users.c.id,],from_obj=t_users)).fetchall()] + goal = target.trigger.goal + achievement = goal.achievement + + for user_id in user_ids: + user_has_level = Achievement.get_level_int(user_id, achievement["id"]) + user_wants_level = min((user_has_level or 0) + 1, achievement["maxlevel"]) + goal_eval = Goal.evaluate(goal, user_id, user_wants_level, None, execute_triggers=False) + + previous_goal = Goal.basic_goal_output(goal, user_wants_level - 1).get("goal_goal", 0) + current_percentage = float(goal_eval["value"]-previous_goal) / float(goal_eval["goal_goal"]-previous_goal) + operator = goal["operator"] + required_percentage = target["condition_percentage"] + + if (operator == "geq" and current_percentage >= required_percentage) \ + or (operator == "leq" and current_percentage <= required_percentage): + GoalTriggerStep.execute( + trigger_step=target, + user_id=user_id, + current_percentage=current_percentage, + value=goal_eval["value"], + goal_goal=goal_eval["goal_goal"], + goal_level=user_wants_level, + goal_properties=Goal.get_properties(goal,user_wants_level), + suppress_actions = True ) - uS.add(m) +def backref(*args,**kw): + if not "passive_deletes" in kw: + kw["passive_deletes"] = True + return sa_backref(*args,**kw) + +def relationship(*args,**kw): + if not "passive_deletes" in kw: + kw["passive_deletes"] = True + if "backref" in kw: + if type(kw["backref"]=="str"): + kw["backref"] = backref(kw["backref"]) + return sa_relationship(*args,**kw) mapper(AuthUser, t_auth_users, properties={ 'roles' : relationship(AuthRole, secondary=t_auth_users_roles, backref="users") @@ -1470,7 +1546,7 @@ def execute(cls, trigger_step, user_id, current_percentage, value, goal_goal, go }) mapper(Group, t_groups, properties={ - 'users' : relationship(User, secondary=t_users_groups, backref="groups"), + 'users' : relationship(User, secondary=t_users_groups, backref="groups"), }) mapper(Variable, t_variables, properties={ diff --git a/gengine/app/views.py b/gengine/app/views.py index 11ed6f8..132fc02 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -48,8 +48,8 @@ def add_or_update_user(request): if not may_update: raise APIError(403, "forbidden", "You may not edit this user.") - if not exists_by_expr(t_users,t_users.c.id==user_id): - raise APIError(403, "forbidden", "The user does not exist. As the user authentication is enabled, you need to create the AuthUser first.") + #if not exists_by_expr(t_users,t_users.c.id==user_id): + # raise APIError(403, "forbidden", "The user does not exist. As the user authentication is enabled, you need to create the AuthUser first.") lat=None From 8b289adbea667d2b32fcb44619837366018d9bb9 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 19 Jul 2016 16:39:33 +0200 Subject: [PATCH 033/176] remove alembic.ini --- alembic.ini | 80 ----------------------------------------------------- debug.py | 0 2 files changed, 80 deletions(-) delete mode 100644 alembic.ini create mode 100644 debug.py diff --git a/alembic.ini b/alembic.ini deleted file mode 100644 index a4130cb..0000000 --- a/alembic.ini +++ /dev/null @@ -1,80 +0,0 @@ -# A generic, single database configuration. - -[alembic] -# path to migration scripts -script_location = gengine/alembic - -# template used to generate migration files -# file_template = %%(rev)s_%%(slug)s - -# max length of characters to apply to the -# "slug" field -#truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false - -# version location specification; this defaults -# to alembic/versions. When using multiple version -# directories, initial revisions must be specified with --version-path -# version_locations = %(here)s/bar %(here)s/bat alembic/versions - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 - -sqlalchemy.url = postgres://user:password@localhost/gengine - - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic,sentry - -[handlers] -keys = console,sentry - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console,sentry -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[logger_sentry] -level = WARN -handlers = console -qualname = sentry.errors -propagate = 0 - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[handler_sentry] -class = raven.handlers.logging.SentryHandler -args = () -level = WARNING -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/debug.py b/debug.py new file mode 100644 index 0000000..e69de29 From 432a30992127ed9b24c417c282ee235e90a5a9aa Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 20 Jul 2016 13:04:59 +0200 Subject: [PATCH 034/176] option to create admin user when initializing the db --- gengine/maintenance/scripts/initializedb.py | 35 ++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index 8cc886e..dcefdbf 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -14,7 +14,7 @@ from sqlalchemy.sql.schema import Table from gengine.app.permissions import perm_global_delete_user, perm_global_increase_value, perm_global_update_user_infos, \ - perm_global_access_admin_ui + perm_global_access_admin_ui, perm_global_read_messages, perm_global_register_device def usage(argv): @@ -97,8 +97,41 @@ def initialize(settings,options): #upgrade command.upgrade(alembic_cfg,'head') + admin_user = options.get("admin_user", False) + admin_password = options.get("admin_password", False) + + if admin_user and admin_password: + create_user(DBSession = DBSession, user=admin_user,password=admin_password) + engine.dispose() +def create_user(DBSession, user, password): + from gengine.app.model import ( + AuthUser, + User, + AuthRole, + AuthRolePermission + ) + user1 = User(id=1, lat=10, lng=50, timezone="Europe/Berlin") + DBSession.add(user1) + DBSession.flush() + + auth_user = AuthUser(user_id=user1.id, email=user, password=password, active=True) + DBSession.add(auth_user) + + auth_role = AuthRole(name="Global Admin") + DBSession.add(auth_role) + + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_access_admin_ui)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_delete_user)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_increase_value)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_update_user_infos)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_read_messages)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_register_device)) + + auth_user.roles.append(auth_role) + DBSession.add(auth_user) + def populate_demo(DBSession): from gengine.app.model import ( From 9fa08b442376be5ab06f0a9824959b7f2b9b1509 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 20 Jul 2016 13:12:44 +0200 Subject: [PATCH 035/176] add tm --- gengine/maintenance/scripts/initializedb.py | 31 +++++++++++---------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index dcefdbf..e068a26 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -112,25 +112,26 @@ def create_user(DBSession, user, password): AuthRole, AuthRolePermission ) - user1 = User(id=1, lat=10, lng=50, timezone="Europe/Berlin") - DBSession.add(user1) - DBSession.flush() + with transaction.manager: + user1 = User(id=1, lat=10, lng=50, timezone="Europe/Berlin") + DBSession.add(user1) + DBSession.flush() - auth_user = AuthUser(user_id=user1.id, email=user, password=password, active=True) - DBSession.add(auth_user) + auth_user = AuthUser(user_id=user1.id, email=user, password=password, active=True) + DBSession.add(auth_user) - auth_role = AuthRole(name="Global Admin") - DBSession.add(auth_role) + auth_role = AuthRole(name="Global Admin") + DBSession.add(auth_role) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_access_admin_ui)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_delete_user)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_increase_value)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_update_user_infos)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_read_messages)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_register_device)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_access_admin_ui)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_delete_user)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_increase_value)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_update_user_infos)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_read_messages)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_register_device)) - auth_user.roles.append(auth_role) - DBSession.add(auth_user) + auth_user.roles.append(auth_role) + DBSession.add(auth_user) def populate_demo(DBSession): From 36b0df96e5702eadfeac20ae045b0e2915d4a912 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 20 Jul 2016 13:15:53 +0200 Subject: [PATCH 036/176] check for existence before creating --- gengine/maintenance/scripts/initializedb.py | 44 ++++++++++++--------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index e068a26..591fde0 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -15,6 +15,7 @@ from gengine.app.permissions import perm_global_delete_user, perm_global_increase_value, perm_global_update_user_infos, \ perm_global_access_admin_ui, perm_global_read_messages, perm_global_register_device +from gengine.base.model import exists_by_expr def usage(argv): @@ -113,25 +114,30 @@ def create_user(DBSession, user, password): AuthRolePermission ) with transaction.manager: - user1 = User(id=1, lat=10, lng=50, timezone="Europe/Berlin") - DBSession.add(user1) - DBSession.flush() - - auth_user = AuthUser(user_id=user1.id, email=user, password=password, active=True) - DBSession.add(auth_user) - - auth_role = AuthRole(name="Global Admin") - DBSession.add(auth_role) - - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_access_admin_ui)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_delete_user)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_increase_value)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_update_user_infos)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_read_messages)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_register_device)) - - auth_user.roles.append(auth_role) - DBSession.add(auth_user) + existing = DBSession.query(AuthUser).filter_by(email=user).first() + if not existing: + try: + user1 = User(id=1, lat=10, lng=50, timezone="Europe/Berlin") + DBSession.add(user1) + DBSession.flush() + + auth_user = AuthUser(user_id=user1.id, email=user, password=password, active=True) + DBSession.add(auth_user) + + auth_role = AuthRole(name="Global Admin") + DBSession.add(auth_role) + + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_access_admin_ui)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_delete_user)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_increase_value)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_update_user_infos)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_read_messages)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_register_device)) + + auth_user.roles.append(auth_role) + DBSession.add(auth_user) + except: + pass def populate_demo(DBSession): From b5403a69a4fce3c67948174976321c078d2672e3 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 20 Jul 2016 13:39:13 +0200 Subject: [PATCH 037/176] documentation: describe new API methods --- docs/rest/index.rst | 50 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/rest/index.rst b/docs/rest/index.rst index 5349dac..f3bd12e 100644 --- a/docs/rest/index.rst +++ b/docs/rest/index.rst @@ -77,4 +77,52 @@ Get a single achievement Level - GET to "/increase_value/{variable_name}/{userId}" - - can be used to check if a user is allowed to use a reward \ No newline at end of file + - can be used to check if a user is allowed to use a reward + +Authentication +============================== + - POST to "/auth/login" + - Parameters in JSON-Body: email, password + - Returns a json body with a token: + .. code:: json + { + "token" : "foobar...." + } + +Register Device (for Push-Messages) +============================== + - POST to "/register_device/{user_id}" + - Parameters in JSON-Body: device_id, push_id, device_os, app_version + - Returns a json body with an ok status, or an error: + .. code:: json + { + "status" : "ok" + } + +Get Messages +============================== + - GET to "/messages/{user_id}" + - Possible GET Parameters: offset + - Limit is always 100 + - Returns a json body with the messages: + .. code:: json + { + "messages" : [{ + "id" : "....", + "text" : "....", + "is_read" : false, + "created_at" : "...." + }] + } + +Set Messages Read +============================== + - POST to "/read_messages/{user_id}" + - Parameters in JSON-Body: message_id + - Sets all messages as read which are at least as old, as the given message + - Returns a json body with an ok status, or an error: + .. code:: json + { + "status" : "ok" + } + From 258b98a04665169ca1a345a2cd5ce37968a26d92 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 20 Jul 2016 14:10:42 +0200 Subject: [PATCH 038/176] hotfix: caching modified data --- gengine/app/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gengine/app/views.py b/gengine/app/views.py index 132fc02..2098842 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -192,6 +192,8 @@ def increase_value(request): Value.increase_value(variable_name, user, value, key) output = _get_progress(achievements_for_user=user, requesting_user=request.user) + output = copy.deepcopy(output) + for aid in list(output["achievements"].keys()): if len(output["achievements"][aid]["new_levels"])>0: if "levels" in output["achievements"][aid]: From 4dace5438598dc43b88cda2d212b949b6fe106d3 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 20 Jul 2016 16:14:31 +0200 Subject: [PATCH 039/176] include new api calls in admin api ui --- gengine/app/static/admin.js | 28 ++++++++++- gengine/app/static/api.js | 33 +++++++++--- gengine/app/templates/admin/index.html | 70 +++++++++++++++++++++++++- gengine/app/views.py | 14 ++++-- 4 files changed, 134 insertions(+), 11 deletions(-) diff --git a/gengine/app/static/admin.js b/gengine/app/static/admin.js index 5dac964..3906c8e 100644 --- a/gengine/app/static/admin.js +++ b/gengine/app/static/admin.js @@ -2,7 +2,9 @@ jQuery().ready(function($) { var defaultcall = "progress"; var fields=["userid","variable","value","key","achievementid","level", - "lat","lon","friends","groups","timezone","country","region","city"]; + "lat","lon","friends","groups","timezone","country","region","city", + "email","password","device_id","push_id","device_os","app_version", + "offset","message_id"]; var api_funcs = { "progress" : { @@ -31,6 +33,30 @@ jQuery().ready(function($) { "fields":["achievementid","level"], "url":"/achievement/{achievementid}/level/{level}", "method":"GET" + }, + "auth_login" : { + "fields":["email","password"], + "url":"/auth/login", + "method":"POST", + "jsonparams":["email","password"] + }, + "register_device" : { + "fields":["userid","device_id","push_id","device_os","app_version"], + "url":"/register_device/{userid}", + "method":"POST", + "jsonparams":["device_id","push_id","device_os","app_version"] + }, + "get_messages" : { + "fields":["userid","offset"], + "url":"/messages/{userid}", + "method":"GET", + "getparams":["offset"] + }, + "set_messages_read" : { + "fields":["userid","message_id"], + "url":"/read_messages/{userid}", + "method":"POST", + "jsonparams":["message_id"] } }; diff --git a/gengine/app/static/api.js b/gengine/app/static/api.js index e9371eb..0040422 100644 --- a/gengine/app/static/api.js +++ b/gengine/app/static/api.js @@ -39,19 +39,23 @@ var setupAPIForm = function($, defaultcall, fields, api_funcs) { var api_settings_url; var api_settings_method; - var api_settings_postparams; + var api_settings_postparams; + var api_settings_jsonparams; + var api_settings_getparams; - var setURL = function(url,method,postparams) { + var setURL = function(url, method, postparams, jsonparams, getparams) { api_settings_url = API_BASE_URL ? API_BASE_URL+url : url; api_settings_method=method; api_settings_postparams = postparams; + api_settings_jsonparams = jsonparams; + api_settings_getparams = getparams; }; var activationfuncs = {}; $.each(api_funcs,function(k,f) { activationfuncs[k] = function() { setActiveFields(f["fields"]); - setURL(f["url"],f["method"],f["postparams"]); + setURL(f["url"],f["method"],f["postparams"],f["jsonparams"],f["getparams"]); }; }); @@ -83,9 +87,14 @@ var setupAPIForm = function($, defaultcall, fields, api_funcs) { var url = api_settings_url; var method = api_settings_method; var postparams = api_settings_postparams; + var jsonparams = api_settings_jsonparams; + var getparams = api_settings_getparams; var ajax_options={}; ajax_options["data"] = {}; + jsondata = {}; + encoded_get_params = []; + for(var i=0; i0) { + ajax_options["data"] = JSON.stringify(jsondata) + } var request = $.ajax(ajax_options); diff --git a/gengine/app/templates/admin/index.html b/gengine/app/templates/admin/index.html index f5fef58..3ec790c 100644 --- a/gengine/app/templates/admin/index.html +++ b/gengine/app/templates/admin/index.html @@ -42,6 +42,12 @@

    Welcome to the Gamification Engine Admin-Area

    + {%if settings_enable_authentication %} + + {% endif %} + + +
    @@ -113,7 +119,47 @@

    Welcome to the Gamification Engine Admin-Area

    - + +
    + + +
    + +
    + + +
    + +
    + + +
    + +
    + + +
    + +
    + + +
    + +
    + + +
    + +
    + + +
    + +
    + + +
    + @@ -148,6 +194,28 @@

    Welcome to the Gamification Engine Admin-Area

    GET to "/achievement/{achievement_id}/level/{level}"

    + + {%if settings_enable_authentication %} + Login +

    + POST to "/auth/login" +

    + {% endif %} + + Register Device +

    + POST to "/register_device/{user_id}" +

    + + Get Messages +

    + GET to "/messages/{user_id}" +

    + + Set Messages Read +

    + POST to "/read_messages/{user_id}" +

    diff --git a/gengine/app/views.py b/gengine/app/views.py index 2098842..cb12b47 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -352,8 +352,16 @@ def register_device(request): @view_config(route_name='get_messages', renderer='json', request_method="GET") def get_messages(request): - user_id = int(request.matchdict["user_id"]) - offset = int(request.GET.get("offset",0)) + try: + user_id = int(request.matchdict["user_id"]) + except: + user_id = None + + try: + offset = int(request.GET.get("offset",0)) + except: + offset = 0 + limit = 100 if asbool(get_settings().get("enable_user_authentication", False)): @@ -399,7 +407,7 @@ def set_messages_read(request): message_id = doc.get("message_id") q = select([t_user_messages.c.id, - t_user_messages.c.created_at], from_obj=message_id).where(and_(t_user_messages.c.id==message_id, + t_user_messages.c.created_at], from_obj=t_user_messages).where(and_(t_user_messages.c.id==message_id, t_user_messages.c.user_id==user_id)) msg = DBSession.execute(q).fetchone() if not msg: From be7ab1a4505e4ddec6a99f074df8ae8df13b2e1a Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 21 Jul 2016 12:15:31 +0200 Subject: [PATCH 040/176] fix import --- gengine/base/model.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gengine/base/model.py b/gengine/base/model.py index e55e53d..522ec40 100644 --- a/gengine/base/model.py +++ b/gengine/base/model.py @@ -6,7 +6,8 @@ from sqlalchemy.sql.functions import func from zope.sqlalchemy.datamanager import mark_changed -from gengine.metadata import DBSession +import gengine.metadata as meta + from gengine.base.cache import cache_general class ABaseMeta(type): @@ -89,7 +90,7 @@ def get_insert_ids_by_result(r): def exists_by_expr(t, expr): # TODO: use exists instead of count q = select([func.count("*").label("c")], from_obj=t).where(expr) - r = DBSession.execute(q).fetchone() + r = meta.DBSession.execute(q).fetchone() if r.c > 0: return True else: @@ -113,6 +114,6 @@ def valid_timezone(timezone): return True def update_connection(): - session = DBSession() + session = meta.DBSession() mark_changed(session) return session From feb59ae8dec15216a10e8695b5d482f36bceaa05 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 21 Jul 2016 14:32:36 +0200 Subject: [PATCH 041/176] add field for additional public user data --- .../3512efb5496d_additional_public_data.py | 28 ++++++++++ gengine/app/model.py | 51 ++++++++++++++++++- gengine/app/views.py | 31 +++++++++-- 3 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 gengine/app/alembic/versions/3512efb5496d_additional_public_data.py diff --git a/gengine/app/alembic/versions/3512efb5496d_additional_public_data.py b/gengine/app/alembic/versions/3512efb5496d_additional_public_data.py new file mode 100644 index 0000000..0136440 --- /dev/null +++ b/gengine/app/alembic/versions/3512efb5496d_additional_public_data.py @@ -0,0 +1,28 @@ +"""additional_public_data + +Revision ID: 3512efb5496d +Revises: +Create Date: 2016-07-21 13:30:45.257569 + +""" + +# revision identifiers, used by Alembic. +revision = '3512efb5496d' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('users', sa.Column('additional_public_data', sa.JSON(), nullable=True)) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('users', 'additional_public_data') + ### end Alembic commands ### diff --git a/gengine/app/model.py b/gengine/app/model.py index 1eade48..e9449a9 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -51,6 +51,7 @@ Column("country", ty.String(), nullable=True, default=None), Column("region", ty.String(), nullable=True, default=None), Column("city", ty.String(), nullable=True, default=None), + Column("additional_public_data", ty.JSON(), nullable=True, default=None), Column('created_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow), ) @@ -418,7 +419,7 @@ def get_cache_expiration_time_for_today(cls,user): return int((tomorrow-today).total_seconds()) @classmethod - def set_infos(cls,user_id,lat,lng,timezone,country,region,city,friends, groups): + def set_infos(cls,user_id,lat,lng,timezone,country,region,city,friends, groups, additional_public_data): """set the user's metadata like friends,location and timezone""" @@ -449,6 +450,7 @@ def set_infos(cls,user_id,lat,lng,timezone,country,region,city,friends, groups): user.country = country user.region = region user.city = city + user.additional_public_data = additional_public_data DBSession.add(user) DBSession.flush() @@ -494,10 +496,57 @@ def delete_user(cls,user_id): update_connection().execute(t_values.delete().where(t_values.c.user_id==user_id)) update_connection().execute(t_users.delete().where(t_users.c.id==user_id)) + @classmethod + def basic_output(cls, user): + return { + "id" : user["id"], + "additional_public_data" : user["additional_public_data"] + } + + @classmethod + def full_output(cls, user_id): + + user = DBSession.execute(t_users.select().where(t_users.c.id == user_id)).fetchone() + + j = t_users.join(t_users_users,t_users_users.c.to_id == t_users.c.id) + friends = DBSession.execute(t_users.select(from_obj=j).where(t_users_users.c.from_id == user_id)).fetchall() + + j = t_groups.join(t_users_groups) + groups = DBSession.execute(t_groups.select(from_obj=j).where(t_users_groups.c.user_id== user_id)).fetchall() + + ret = { + "id" : user["id"], + "lat" : user["lat"], + "lng" : user["lng"], + "timezone" : user["timezone"], + "country": user["country"], + "region": user["region"], + "city": user["city"], + "created_at": user["created_at"], + "additional_public_data": user["additional_public_data"], + "friends" : [User.basic_output(f) for f in friends], + "groups": [Group.basic_output(g) for g in groups], + } + + if get_settings().get("enable_user_authentication"): + auth_user = DBSession.execute(t_auth_users.select().where(t_auth_users.c.user_id == user_id)).fetchone() + + ret.update({ + "email" : auth_user["email"] + }) + + return ret + class Group(ABase): def __unicode__(self, *args, **kwargs): return "(ID: %s)" % (self.id,) + @classmethod + def basic_output(cls, group): + return { + "id" : group["id"] + } + class Variable(ABase): """A Variable is anything that should be meassured in your application and be used in :class:`.Goal`. diff --git a/gengine/app/views.py b/gengine/app/views.py index cb12b47..6152b7e 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -7,6 +7,9 @@ import base64 import copy import datetime + +import json + from pyramid.request import Request from pyramid.response import Response from pyramid.settings import asbool @@ -86,7 +89,15 @@ def add_or_update_user(request): city=None if len(request.POST.get("city",""))>0: city = request.POST["city"] - + + additional_public_data = {} + if len(request.POST.get("additional_public_data", "")) > 0: + try: + additional_public_data = json.loads(request.POST["additional_public_data"]) + except: + additional_public_data = {} + + User.set_infos(user_id=user_id, lat=lat, lng=lon, @@ -95,9 +106,10 @@ def add_or_update_user(request): region=region, city=city, friends=friends, - groups=groups) + groups=groups, + additional_public_data = additional_public_data) - return {"status" : "OK"} + return {"status" : "OK", "user" : User.full_output(user_id)} @view_config(route_name='delete_user', renderer='string', request_method="DELETE") def delete_user(request): @@ -161,7 +173,14 @@ def get_progress(request): if not user: raise APIError(404, "user_not_found", "user not found") - return _get_progress(achievements_for_user=user, requesting_user=request.user) + output = _get_progress(achievements_for_user=user, requesting_user=request.user) + output = copy.deepcopy(output) + + for aid in list(output["achievements"].keys()): + if "new_levels" in output["achievements"][aid]: + del output["achievements"][aid]["new_levels"] + + return output @view_config(route_name='increase_value', renderer='json', request_method="POST") @view_config(route_name='increase_value_with_key', renderer='json', request_method="POST") @@ -238,6 +257,7 @@ def increase_multi_values(request): Value.increase_value(variable_name, user, value, key) output = _get_progress(achievements_for_user=user, requesting_user=request.user) + output = copy.deepcopy(output) for aid in list(output["achievements"].keys()): if len(output["achievements"][aid]["new_levels"])>0: @@ -309,7 +329,8 @@ def auth_login(request): DBSession.commit() return { - "token" : token + "token" : token, + "user" : User.full_output(user.user_id), } @view_config(route_name='register_device', renderer='json', request_method="POST") From 3dc9a83879d5dc8e73914c10bca60324f403a98c Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 21 Jul 2016 15:20:02 +0200 Subject: [PATCH 042/176] unset expire_on_commit --- gengine/metadata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gengine/metadata.py b/gengine/metadata.py index 2a6d69f..8f3156a 100644 --- a/gengine/metadata.py +++ b/gengine/metadata.py @@ -23,8 +23,7 @@ def init_session(override_session=None): DBSession = scoped_session( sessionmaker( extension=ZopeTransactionExtension(), - class_=MySession, - expire_on_commit=False + class_=MySession ) ) From 41729d7ec601ccc4b9bea6ef6a507e25d14fa174 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 21 Jul 2016 15:41:51 +0200 Subject: [PATCH 043/176] multiple calls to init_session / init_base should not be executed --- gengine/metadata.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gengine/metadata.py b/gengine/metadata.py index 8f3156a..8d5ee82 100644 --- a/gengine/metadata.py +++ b/gengine/metadata.py @@ -17,6 +17,8 @@ def rollback(self,*args,**kw): def init_session(override_session=None): global DBSession + if DBSession: + return if override_session: DBSession = override_session else: @@ -31,6 +33,8 @@ def init_session(override_session=None): def init_declarative_base(override_base=None): global Base + if Base: + return if override_base: Base=override_base else: @@ -47,5 +51,4 @@ def init_declarative_base(override_base=None): def init_db(engine): DBSession.configure(bind=engine) Base.metadata.bind = engine - \ No newline at end of file From 96e2fce73eb73d29bf7b3a9b7957cd90a3cf7119 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 21 Jul 2016 15:49:38 +0200 Subject: [PATCH 044/176] no commit here! --- gengine/app/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gengine/app/views.py b/gengine/app/views.py index 6152b7e..621d530 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -326,7 +326,6 @@ def auth_login(request): ) DBSession.add(tokenObj) - DBSession.commit() return { "token" : token, From 971dba38559f0fda63aedaec891af768b9d76378 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 21 Jul 2016 16:02:35 +0200 Subject: [PATCH 045/176] add additional_public_data to api ui --- gengine/app/static/admin.js | 6 +++--- gengine/app/templates/admin/index.html | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/gengine/app/static/admin.js b/gengine/app/static/admin.js index 3906c8e..3311a1e 100644 --- a/gengine/app/static/admin.js +++ b/gengine/app/static/admin.js @@ -4,7 +4,7 @@ jQuery().ready(function($) { var fields=["userid","variable","value","key","achievementid","level", "lat","lon","friends","groups","timezone","country","region","city", "email","password","device_id","push_id","device_os","app_version", - "offset","message_id"]; + "offset","message_id","additional_public_data"]; var api_funcs = { "progress" : { @@ -19,10 +19,10 @@ jQuery().ready(function($) { "postparams":["value"] }, "add_or_update_user" : { - "fields":["userid","lat","lon","friends","groups","timezone","country","region","city"], + "fields":["userid","lat","lon","friends","groups","timezone","country","region","city","additional_public_data"], "url":"/add_or_update_user/{userid}", "method":"POST", - "postparams":["lat","lon","friends","groups","timezone","country","region","city"] + "postparams":["lat","lon","friends","groups","timezone","country","region","city","additional_public_data"] }, "delete_user" : { "fields":["userid"], diff --git a/gengine/app/templates/admin/index.html b/gengine/app/templates/admin/index.html index 3ec790c..7716178 100644 --- a/gengine/app/templates/admin/index.html +++ b/gengine/app/templates/admin/index.html @@ -160,6 +160,11 @@

    Welcome to the Gamification Engine Admin-Area

    +
    + + +
    + From 44532cf61949dfb7a3e22e35ee7a5f4e7dfd510c Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 22 Jul 2016 13:48:39 +0200 Subject: [PATCH 046/176] work on pushes --- gengine/__init__.py | 2 +- gengine/app/admin.py | 5 +++-- gengine/app/model.py | 12 ++++++++++++ gengine/app/templates/admin_layout.html | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/gengine/__init__.py b/gengine/__init__.py index 701bce5..29c085b 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -103,7 +103,7 @@ def has_perm(request, name): config.include(config_app_routes, route_prefix=urlprefix) - config.add_route('admin_app', '/*subpath') #prefix is set in flaskadmin.py + config.add_route('admin_app', '/admin/*subpath') from gengine.app.admin import init_admin as init_tenantadmin init_tenantadmin(urlprefix=urlprefix, diff --git a/gengine/app/admin.py b/gengine/app/admin.py index f289c4e..ae2e6b6 100644 --- a/gengine/app/admin.py +++ b/gengine/app/admin.py @@ -69,13 +69,14 @@ def init_admin(urlprefix="",secret="fKY7kJ2xSrbPC5yieEjV",override_admin=None,ov @adminapp.context_processor def inject_version(): return { "gamification_engine_version" : pkg_resources.get_distribution("gamification-engine").version, - "settings_enable_authentication" : asbool(get_settings().get("enable_user_authentication",False))} + "settings_enable_authentication" : asbool(get_settings().get("enable_user_authentication",False)), + "urlprefix" : get_settings().get("urlprefix","/")} if not override_admin: admin = Admin(adminapp, name="Gamification Engine - Admin Control Panel", base_template='admin_layout.html', - url=urlprefix+"/admin" + url=urlprefix+"" ) else: admin = override_admin diff --git a/gengine/app/model.py b/gengine/app/model.py index e9449a9..f444ffc 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1493,6 +1493,17 @@ def get_text(cls, row): def text(self): return Translation.trs(self.translation_id, self.params) + @classmethod + def deliver(cls, message): + from gengine.app.push import send_push_message + + send_push_message( + user_id = message["user_id"], + text = UserMessage.get_text(message), + custom_payload = {}, + title = get_settings().get("push_title","Gamification-Engine") + ) + class GoalTrigger(ABase): def __unicode__(self, *args, **kwargs): return "GoalTrigger: %s" % (self.id,) @@ -1522,6 +1533,7 @@ def execute(cls, trigger_step, user_id, current_percentage, value, goal_goal, go is_read = False, ) uS.add(m) + UserMessage.deliver(m) @event.listens_for(GoalTriggerStep, "after_insert") @event.listens_for(GoalTriggerStep, 'after_update') diff --git a/gengine/app/templates/admin_layout.html b/gengine/app/templates/admin_layout.html index 9e064e1..f8aadaa 100644 --- a/gengine/app/templates/admin_layout.html +++ b/gengine/app/templates/admin_layout.html @@ -5,7 +5,7 @@ {{ super() }} {% endblock %} From 85a388333de934d3fb36f5974f8d2313cb384a60 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 22 Jul 2016 14:06:24 +0200 Subject: [PATCH 047/176] add user language --- .../d4a70083f72e_add_user_language.py | 34 +++++++++++++++++ gengine/app/model.py | 37 +++++++++++++------ gengine/app/static/admin.js | 6 +-- gengine/app/templates/admin/index.html | 5 +++ gengine/app/views.py | 5 +++ 5 files changed, 73 insertions(+), 14 deletions(-) create mode 100644 gengine/app/alembic/versions/d4a70083f72e_add_user_language.py diff --git a/gengine/app/alembic/versions/d4a70083f72e_add_user_language.py b/gengine/app/alembic/versions/d4a70083f72e_add_user_language.py new file mode 100644 index 0000000..5359c0a --- /dev/null +++ b/gengine/app/alembic/versions/d4a70083f72e_add_user_language.py @@ -0,0 +1,34 @@ +"""Add User Language + +Revision ID: d4a70083f72e +Revises: 3512efb5496d +Create Date: 2016-07-22 14:04:43.900826 + +""" + +# revision identifiers, used by Alembic. +revision = 'd4a70083f72e' +down_revision = '3512efb5496d' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_index(op.f('ix_languages_name'), 'languages', ['name'], unique=False) + op.create_index(op.f('ix_translationvariables_name'), 'translationvariables', ['name'], unique=False) + op.add_column('users', sa.Column('language_id', sa.Integer(), nullable=True)) + op.create_foreign_key(op.f('fk_users_language_id_languages'), 'users', 'languages', ['language_id'], ['id']) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(op.f('fk_users_language_id_languages'), 'users', type_='foreignkey') + op.drop_column('users', 'language_id') + op.drop_index(op.f('ix_translationvariables_name'), table_name='translationvariables') + op.drop_index(op.f('ix_languages_name'), table_name='languages') + ### end Alembic commands ### diff --git a/gengine/app/model.py b/gengine/app/model.py index f444ffc..0426a33 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -47,6 +47,7 @@ Column('id', ty.BigInteger, primary_key = True), Column("lat", ty.Float(Precision=64), nullable=True), Column("lng", ty.Float(Precision=64), nullable=True), + Column("language_id", ty.Integer, ForeignKey("languages.id"), nullable=True), Column("timezone", ty.String(), nullable=False, default="UTC"), Column("country", ty.String(), nullable=True, default=None), Column("region", ty.String(), nullable=True, default=None), @@ -226,12 +227,12 @@ def get_default_token_valid_time(): t_languages = Table('languages', Base.metadata, Column('id', ty.Integer, primary_key = True), - Column('name', ty.String(255), nullable = False), + Column('name', ty.String(255), nullable = False, index=True), ) t_translationvariables = Table('translationvariables', Base.metadata, Column('id', ty.Integer, primary_key = True), - Column('name', ty.String(255), nullable = False), + Column('name', ty.String(255), nullable = False, index=True), ) t_translations = Table('translations', Base.metadata, @@ -419,7 +420,7 @@ def get_cache_expiration_time_for_today(cls,user): return int((tomorrow-today).total_seconds()) @classmethod - def set_infos(cls,user_id,lat,lng,timezone,country,region,city,friends, groups, additional_public_data): + def set_infos(cls,user_id,lat,lng,timezone,country,region,city,language,friends, groups, additional_public_data): """set the user's metadata like friends,location and timezone""" @@ -452,6 +453,12 @@ def set_infos(cls,user_id,lat,lng,timezone,country,region,city,friends, groups, user.city = city user.additional_public_data = additional_public_data + language = DBSession.execute(t_languages.select().where(t_languages.c.name == language)).fetchone() + if language: + user.language_id = language["id"] + else: + user.language_id = None + DBSession.add(user) DBSession.flush() @@ -519,6 +526,7 @@ def full_output(cls, user_id): "lat" : user["lat"], "lng" : user["lng"], "timezone" : user["timezone"], + "language": user["language"], "country": user["country"], "region": user["region"], "city": user["city"], @@ -1068,7 +1076,7 @@ class Goal(ABase): def __unicode__(self, *args, **kwargs): if self.name_translation_id!=None: - name = Translation.trs(self.name_translation.id, {"level":1, "goal":'0'})[_fallback_language] + name = Translation.trs(self.name_translation.id, {"level":1, "goal":'0'})[get_settings().get("fallback_language","en")] return str(name) + " (ID: %s)" % (self.id,) else: return self.name + " (ID: %s)" % (self.id,) @@ -1439,7 +1447,6 @@ class TranslationVariable(ABase): def __unicode__(self, *args, **kwargs): return "%s" % (self.name,) -_fallback_language="en" class Translation(ABase): def __unicode__(self, *args, **kwargs): return "%s" % (self.text,) @@ -1457,14 +1464,14 @@ def trs(cls,translation_id,params={}): ret = {str(x["name"]) : evaluate_string(x["text"],params) for x in cls.get_translation_variable(translation_id)} except Exception as e: ret = {str(x["name"]) : x["text"] for x in cls.get_translation_variable(translation_id)} - log.exception("Evaluation of string-forumlar failed: %s" % (ret.get(_fallback_language,translation_id),)) + log.exception("Evaluation of string-forumlar failed: %s" % (ret.get(get_settings().get("fallback_language","en"),translation_id),)) - if not _fallback_language in ret: - ret[_fallback_language] = "[not_translated]_"+str(translation_id) + if not get_settings().get("fallback_language","en") in ret: + ret[get_settings().get("fallback_language","en")] = "[not_translated]_"+str(translation_id) for lang in cls.get_languages(): if not str(lang["name"]) in ret: - ret[str(lang["name"])] = ret[_fallback_language] + ret[str(lang["name"])] = ret[get_settings().get("fallback_language","en")] return ret @@ -1483,7 +1490,7 @@ def get_languages(cls): class UserMessage(ABase): def __unicode__(self, *args, **kwargs): - return "Message: %s" % (Translation.trs(self.translation_id,self.params).get(_fallback_language),) + return "Message: %s" % (Translation.trs(self.translation_id,self.params).get(get_settings().get("fallback_language","en")),) @classmethod def get_text(cls, row): @@ -1497,9 +1504,17 @@ def text(self): def deliver(cls, message): from gengine.app.push import send_push_message + text = UserMessage.get_text(message) + language = get_settings().get("fallback_language", "en") + j = t_users.join(t_languages) + user_language = DBSession.execute(select([t_languages.c.name],from_obj=j).where(t_users.c.id==message["user_id"])).fetchone() + if user_language: + language = user_language["name"] + translated_text = text[language] + send_push_message( user_id = message["user_id"], - text = UserMessage.get_text(message), + text = translated_text, custom_payload = {}, title = get_settings().get("push_title","Gamification-Engine") ) diff --git a/gengine/app/static/admin.js b/gengine/app/static/admin.js index 3311a1e..7f840c0 100644 --- a/gengine/app/static/admin.js +++ b/gengine/app/static/admin.js @@ -4,7 +4,7 @@ jQuery().ready(function($) { var fields=["userid","variable","value","key","achievementid","level", "lat","lon","friends","groups","timezone","country","region","city", "email","password","device_id","push_id","device_os","app_version", - "offset","message_id","additional_public_data"]; + "offset","message_id","additional_public_data","language"]; var api_funcs = { "progress" : { @@ -19,10 +19,10 @@ jQuery().ready(function($) { "postparams":["value"] }, "add_or_update_user" : { - "fields":["userid","lat","lon","friends","groups","timezone","country","region","city","additional_public_data"], + "fields":["userid","lat","lon","friends","groups","timezone","country","region","city","additional_public_data","language"], "url":"/add_or_update_user/{userid}", "method":"POST", - "postparams":["lat","lon","friends","groups","timezone","country","region","city","additional_public_data"] + "postparams":["lat","lon","friends","groups","timezone","country","region","city","additional_public_data","language"] }, "delete_user" : { "fields":["userid"], diff --git a/gengine/app/templates/admin/index.html b/gengine/app/templates/admin/index.html index 7716178..181ef96 100644 --- a/gengine/app/templates/admin/index.html +++ b/gengine/app/templates/admin/index.html @@ -104,6 +104,11 @@

    Welcome to the Gamification Engine Admin-Area

    + +
    + + +
    diff --git a/gengine/app/views.py b/gengine/app/views.py index 621d530..1235cb8 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -90,6 +90,10 @@ def add_or_update_user(request): if len(request.POST.get("city",""))>0: city = request.POST["city"] + language = None + if len(request.POST.get("language", "")) > 0: + language= request.POST["language"] + additional_public_data = {} if len(request.POST.get("additional_public_data", "")) > 0: try: @@ -105,6 +109,7 @@ def add_or_update_user(request): country=country, region=region, city=city, + language=language, friends=friends, groups=groups, additional_public_data = additional_public_data) From 813f6614a2d9933fcf766fa4c37d92c560f27e13 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 22 Jul 2016 15:40:29 +0200 Subject: [PATCH 048/176] correct language output --- gengine/app/model.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 0426a33..2cc7876 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -521,12 +521,18 @@ def full_output(cls, user_id): j = t_groups.join(t_users_groups) groups = DBSession.execute(t_groups.select(from_obj=j).where(t_users_groups.c.user_id== user_id)).fetchall() + language = get_settings().get("fallback_language","en") + j = t_users.join(t_languages) + user_language = DBSession.execute(select([t_languages.c.name], from_obj=j).where(t_users.c.id == user_id)).fetchone() + if user_language: + language = user_language["name"] + ret = { "id" : user["id"], "lat" : user["lat"], "lng" : user["lng"], "timezone" : user["timezone"], - "language": user["language"], + "language": language, "country": user["country"], "region": user["region"], "city": user["city"], From 8d5490dfe2bd557988c64fcf118006bb1b88c260 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 22 Jul 2016 15:50:54 +0200 Subject: [PATCH 049/176] distinct push ids --- gengine/app/push.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/app/push.py b/gengine/app/push.py index 3cfc28c..86f1e8e 100644 --- a/gengine/app/push.py +++ b/gengine/app/push.py @@ -183,7 +183,7 @@ def send_push_message( if not android_text: android_text = text - rows = DBSession.execute(t_user_device.select().where(t_user_device.c.user_id==user_id)).fetchall() + rows = DBSession.execute(select([t_user_device.c.push_id, t_user_device.c.device_os], from_obj=t_user_device).distinct().where(t_user_device.c.user_id==user_id)).fetchall() for device in rows: From 7a4131b311e05656bf61d1d869398057a054336e Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 26 Jul 2016 22:16:46 +0200 Subject: [PATCH 050/176] fix output when the authUser does not exist yet --- gengine/app/model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 2cc7876..8d2e2c1 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -544,10 +544,10 @@ def full_output(cls, user_id): if get_settings().get("enable_user_authentication"): auth_user = DBSession.execute(t_auth_users.select().where(t_auth_users.c.user_id == user_id)).fetchone() - - ret.update({ - "email" : auth_user["email"] - }) + if auth_user: + ret.update({ + "email" : auth_user["email"] + }) return ret From 3ef9ddfb189265d6e698178bb77633e2be659ce2 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 26 Jul 2016 22:16:56 +0200 Subject: [PATCH 051/176] fix gcm error handling --- gengine/app/push.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/gengine/app/push.py b/gengine/app/push.py index 86f1e8e..342eff6 100644 --- a/gengine/app/push.py +++ b/gengine/app/push.py @@ -128,20 +128,8 @@ def gcm_feedback(response): if error in ['NotRegistered', 'InvalidRegistration']: # Remove reg_ids from database for reg_id in reg_ids: - - q = t_user_device.select().where(t_user_device.c.push_id == reg_id) - rows = uS.execute(q).fetchall() - - for device in rows: - if not isinstance(reg_id, str): - reg_id = reg_id.decode("utf8") - - log.debug("Removing reg_id: {0} from db".format(reg_id)) - - uS.execute(t_user_device.delete().where( - t_user_device.c.device_id == device["device_id"], - t_user_device.c.user_id == device["user_id"], - )) + q = t_user_device.delete().where(t_user_device.c.push_id == reg_id) + uS.execute(q) # Repace reg_id with canonical_id in your database if 'canonical' in response: @@ -153,9 +141,7 @@ def gcm_feedback(response): q = t_user_device.update().values({ "push_id" : canonical_id - }).where(and_( - t_user_device.c.push_id == reg_id, - )) + }).where(t_user_device.c.push_id == reg_id) uS.execute(q) From a4c5ea89de99f6868629865460344587cef5d066 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 4 Oct 2016 11:57:39 +0200 Subject: [PATCH 052/176] typo --- gengine/app/admin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gengine/app/admin.py b/gengine/app/admin.py index ae2e6b6..2f0a79e 100644 --- a/gengine/app/admin.py +++ b/gengine/app/admin.py @@ -24,7 +24,7 @@ admin=None -def resole_uri(uri): +def resolve_uri(uri): from pyramid.path import PkgResourcesAssetDescriptor pkg_name,path=uri.split(":",1) a = PkgResourcesAssetDescriptor(pkg_name,path) @@ -32,6 +32,8 @@ def resole_uri(uri): absolute = os.path.abspath(absolute) #so we make it absolute return absolute +resole_uri = resolve_uri # there was a typing error once... + def get_static_view(folder,flaskadminapp): folder=resole_uri(folder) From 0b1a975a07303a840248f493b4a5ee64c2fcc563 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 4 Oct 2016 11:58:15 +0200 Subject: [PATCH 053/176] fix a py3 bug concerning datetime aware rules --- gengine/app/model.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 8d2e2c1..82867df 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1125,9 +1125,8 @@ def generate_statement_cache(): j = t_values.join(t_variables) - if evaluation_type in ("daily","weekly","monthly","yearly"): - # We need to access the user's timezone later - j = j.join(t_users) + # # We need to access the user's timezone later + j = j.join(t_users) datetime_col=None if group_by_dateformat: @@ -1162,11 +1161,11 @@ def generate_statement_cache(): elif evaluation_type=="yearly": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("year","users.timezone"))) - if datetime_col or group_by_key: - if datetime_col: + if datetime_col is not None or group_by_key is not None: + if datetime_col is not None: q = q.group_by(datetime_col) - if group_by_key: + if group_by_key is not None: q = q.group_by(t_values.c.key) query_with_groups = q.alias() @@ -1571,6 +1570,9 @@ def insert_trigger_step_executions_after_step_upsert(mapper,connection,target): goal_eval = Goal.evaluate(goal, user_id, user_wants_level, None, execute_triggers=False) previous_goal = Goal.basic_goal_output(goal, user_wants_level - 1).get("goal_goal", 0) + if previous_goal == goal_eval["goal_goal"]: + previous_goal = 0.0 + current_percentage = float(goal_eval["value"]-previous_goal) / float(goal_eval["goal_goal"]-previous_goal) operator = goal["operator"] required_percentage = target["condition_percentage"] From 4376cd70637ced102ef101a6703cdfefacd9b080 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 5 Oct 2016 16:29:02 +0200 Subject: [PATCH 054/176] Move evaluation type into achievement table to implement repeating achievements --- gengine/app/admin.py | 4 +- gengine/app/model.py | 152 ++++++++++++++++++++++++++++--------------- gengine/app/views.py | 96 ++++++++++++++++++--------- 3 files changed, 167 insertions(+), 85 deletions(-) diff --git a/gengine/app/admin.py b/gengine/app/admin.py index 2f0a79e..fe99121 100644 --- a/gengine/app/admin.py +++ b/gengine/app/admin.py @@ -131,7 +131,7 @@ def __init__(self, session, **kwargs): super(ModelViewAchievementCategory, self).__init__(AchievementCategory, session, **kwargs) class ModelViewAchievement(ModelView): - column_list = ('name','valid_start','valid_end','relevance') + column_list = ('name','evaluation','valid_start','valid_end','relevance') column_searchable_list = ('name',) form_excluded_columns =('rewards','users','goals','properties','updated_at') fast_mass_delete = True @@ -168,7 +168,7 @@ def __init__(self, session, **kwargs): super(ModelViewGoalTrigger, self).__init__(GoalTrigger, session, **kwargs) class ModelViewGoal(ModelView): - column_list = ('condition','evaluation','operator','goal','timespan','priority','achievement','updated_at') + column_list = ('condition','operator','goal','timespan','priority','achievement','updated_at') form_excluded_columns =('properties','triggers') #column_searchable_list = ('name',) column_filters = (Achievement.id,) diff --git a/gengine/app/model.py b/gengine/app/model.py index 82867df..371b839 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -123,8 +123,10 @@ def get_default_token_valid_time(): Column("lng", ty.Float(Precision=64), nullable=True), Column("max_distance", ty.Integer, nullable=True), Column('priority', ty.Integer, index=True, default=0), + Column('evaluation',ty.Enum("immediately","daily","weekly","monthly","yearly","end", name="evaluation_types"), default="immediately", nullable=False), Column('relevance',ty.Enum("friends","city","own", name="relevance_types"), default="own"), Column('view_permission',ty.Enum("everyone", "own", name="achievement_view_permission"), default="everyone"), + Column('created_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow), ) t_goals = Table("goals", Base.metadata, @@ -133,7 +135,6 @@ def get_default_token_valid_time(): Column('name_translation_id', ty.Integer, ForeignKey("translationvariables.id", ondelete="RESTRICT"), nullable = True), #TODO: deprecate name_translation Column('condition', ty.String(255), nullable=True), - Column('evaluation',ty.Enum("immediately","daily","weekly","monthly","yearly","end", name="evaluation_types")), Column('timespan',ty.Integer, nullable=True), Column('group_by_key', ty.Boolean(), default=False), Column('group_by_dateformat', ty.String(255), nullable=True), @@ -146,6 +147,7 @@ def get_default_token_valid_time(): t_goal_evaluation_cache = Table("goal_evaluation_cache", Base.metadata, Column("goal_id", ty.Integer, ForeignKey("goals.id", ondelete="CASCADE"), primary_key=True, nullable=False), + Column('achievement_date', ty.DateTime, nullable=True), # To identify the goals for monthly, weekly, ... achievements; Column("user_id", ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), primary_key=True, nullable=False), Column("achieved", ty.Boolean), Column("value", ty.Float), @@ -211,8 +213,9 @@ def get_default_token_valid_time(): t_achievements_users = Table('achievements_users', Base.metadata, Column('user_id', ty.BigInteger, ForeignKey("users.id"), primary_key = True, index=True, nullable=False), Column('achievement_id', ty.Integer, ForeignKey("achievements.id", ondelete="CASCADE"), primary_key = True, nullable=False), + Column('achievement_date', ty.DateTime, nullable=True, primary_key=True), Column('level', ty.Integer, primary_key = True, default=1), - Column('updated_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow), + Column('updated_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, index=True), ) t_requirements = Table('requirements', Base.metadata, @@ -578,18 +581,14 @@ def get_variable_by_name(cls,name): @classmethod def get_datetime_for_tz_and_group(cls,tz,group): - """get the datetime of the current row, needed for grouping - - when "timezone" is used as a group name, the values are grouped to the nearest time in (09:00, 12:00, 15:00, 18:00, 21:00) - (timezone to use is given as parameter) - """ + """get the datetime of the current row, needed for grouping""" tzobj = pytz.timezone(tz) now = datetime.datetime.now(tzobj) #now = now.replace(tzinfo=pytz.utc) t = None if group=="year": - t = now.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0 ) + t = now.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0) elif group=="month": t = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) elif group=="week": @@ -628,10 +627,11 @@ def map_variables_to_rules(cls): def invalidate_caches_for_variable_and_user(cls,variable_id,user_id): """ invalidate the relevant caches for this user and all relevant users with concerned leaderboards""" goalsandachievements = cls.map_variables_to_rules().get(variable_id,[]) - - Goal.clear_goal_caches(user_id, [entry["goal"]["id"] for entry in goalsandachievements]) + timezone = User.get_user(user_id)["timezone"] + Goal.clear_goal_caches(user_id, [(entry["goal"]["id"],Achievement.get_datetime_for_evaluation_type(timezone, entry["achievement"]["evaluation"])) for entry in goalsandachievements]) for entry in goalsandachievements: - Achievement.invalidate_evaluate_cache(user_id,entry["achievement"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(timezone, entry["achievement"]["evaluation"]) + Achievement.invalidate_evaluate_cache(user_id, entry["achievement"], achievement_date) @classmethod def may_increase(cls, variable_row, request, user_id): @@ -785,22 +785,24 @@ def get_relevant_users_by_achievement_and_user_reverse(cls,achievement,user_id): return set(users) @classmethod - def get_level(cls, user_id, achievement_id): + def get_level(cls, user_id, achievement_id, achievement_date): """get the current level of the user for this achievement.""" def generate(): q = select([t_achievements_users.c.level, + t_achievements_users.c.achievement_date, t_achievements_users.c.updated_at], and_(t_achievements_users.c.user_id==user_id, + t_achievements_users.c.achievement_date== achievement_date, t_achievements_users.c.achievement_id==achievement_id)).order_by(t_achievements_users.c.level.desc()) return [x for x in DBSession.execute(q).fetchall()] - return cache_achievements_users_levels.get_or_create("%s_%s" % (user_id,achievement_id),generate) + return cache_achievements_users_levels.get_or_create("%s_%s_%s" % (user_id,achievement_id,achievement_date),generate) @classmethod - def get_level_int(cls,user_id,achievement_id): + def get_level_int(cls,user_id,achievement_id,achievement_date): """get the current level of the user for this achievement as int (0 if the user does not have this achievement)""" - lvls = Achievement.get_level(user_id, achievement_id) + lvls = Achievement.get_level(user_id, achievement_id,achievement_date) if not lvls: return 0 @@ -853,18 +855,19 @@ def basic_output(cls,achievement,goals,include_levels=True, return out @classmethod - def evaluate(cls, user, achievement_id): + def evaluate(cls, user, achievement_id, achievement_date): """evaluate the achievement including all its subgoals for the user. return the basic_output for the achievement plus information about the new achieved levels """ def generate(): - user_id = user["id"] achievement = Achievement.get_achievement(achievement_id) + + user_id = user["id"] user_ids = Achievement.get_relevant_users_by_achievement_and_user(achievement, user_id) - user_has_level = Achievement.get_level_int(user_id, achievement["id"]) + user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) user_wants_level = min((user_has_level or 0)+1, achievement["maxlevel"]) goal_evals={} @@ -873,13 +876,13 @@ def generate(): for goal in goals: - goal_eval = Goal.get_goal_eval_cache(goal["id"], user_id) + goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) if not goal_eval: - Goal.evaluate(goal, user_id, user_wants_level,None) - goal_eval = Goal.get_goal_eval_cache(goal["id"], user_id) + Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level,None) + goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) if achievement["relevance"]=="friends" or achievement["relevance"]=="city": - goal_eval["leaderboard"] = Goal.get_leaderboard(goal, user_ids) + goal_eval["leaderboard"] = Goal.get_leaderboard(goal, achievement, achievement_date, user_ids) goal_eval["leaderboard_position"] = list(filter(lambda x : x["user_id"]==user_id, goal_eval["leaderboard"]))[0]["position"] goal_evals[goal["id"]]=goal_eval @@ -926,12 +929,12 @@ def generate(): })) #invalidate getter - cache_achievements_users_levels.delete("%s_%s" % (user_id,achievement_id)) + cache_achievements_users_levels.delete("%s_%s_%s" % (user_id,achievement_id,achievement_date)) user_has_level = user_wants_level user_wants_level = user_wants_level+1 - Goal.clear_goal_caches(user_id, [g["goal_id"] for g in goal_evals.values()]) + Goal.clear_goal_caches(user_id, [(g["goal_id"],achievement_date) for g in goal_evals.values()]) #the level has been updated, we need to do recursion now... #but only if there are more levels... if user_has_level < achievement["maxlevel"]: @@ -943,11 +946,12 @@ def generate(): output.update({ "level" : user_has_level, "levels_achieved" : { - str(x["level"]) : x["updated_at"] for x in Achievement.get_level(user_id, achievement["id"]) + str(x["level"]) : x["updated_at"] for x in Achievement.get_level(user_id, achievement["id"], achievement_date) }, "maxlevel" : achievement["maxlevel"], "new_levels" : {}, "goals":goal_evals, + "achievement_date": achievement_date, #"updated_at":combine_updated_at([achievement["updated_at"],] + [g["updated_at"] for g in goal_evals]) }) @@ -956,15 +960,16 @@ def generate(): return output - return cache_achievement_eval.get_or_create("%s_%s" % (user["id"],achievement_id),generate) + #TODO ACHIEVEMENT + return cache_achievement_eval.get_or_create("%s_%s_%s" % (user["id"],achievement_id,achievement_date),generate) @classmethod - def invalidate_evaluate_cache(cls,user_id,achievement): + def invalidate_evaluate_cache(cls,user_id,achievement, achievement_date): """invalidate the evaluation cache for all goals of this achievement for the user.""" #We neeed to invalidate for all relevant users because of the leaderboards for uid in Achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user_id): - cache_achievement_eval.delete("%s_%s" % (uid,achievement["id"])) + cache_achievement_eval.delete("%s_%s_%s" % (uid, achievement["id"], achievement_date)) @classmethod @cache_general.cache_on_arguments() @@ -1021,6 +1026,37 @@ def get_achievement_properties(cls,achievement_id,level): .order_by(t_achievements_achievementproperties.c.from_level))\ .fetchall() + @classmethod + def get_datetime_for_evaluation_type(cls, tz, evaluation_type, dt=None): + """ + This computes the datetime to identify the time of the achievement. + Only relevant for repeating achievements (monthly, yearly, weekly, daily) + Returns None for all other achievement types + """ + tzobj = pytz.timezone(tz) + + if not dt: + dt = datetime.datetime.now(tzobj) + else: + dt = dt.astimezone(tzobj) + + t = None + if evaluation_type == "yearly": + t = dt.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0) + elif evaluation_type == "monthly": + t = dt.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + elif evaluation_type == "weekly": + t = dt - datetime.timedelta(days=dt.weekday()) + t = t.replace(hour=0, minute=0, second=0, microsecond=0) + elif evaluation_type == "daily": + t = dt.replace(hour=0, minute=0, second=0, microsecond=0) + elif evaluation_type == "immediately": + return None + elif evaluation_type == "end": + return None + + return t.astimezone(tzobj) + class AchievementProperty(ABase): """A AchievementProperty describes the :class:`Achievement`s of our system. @@ -1093,7 +1129,7 @@ def get_goals(cls,achievement_id): return DBSession.execute(t_goals.select(t_goals.c.achievement_id==achievement_id)).fetchall() @classmethod - def compute_progress(cls,goal,user_id): + def compute_progress(cls, goal, achievement, user_id): """computes the progress of the goal for the given user_id goal attributes: @@ -1117,7 +1153,7 @@ def generate_statement_cache(): group_by_key = goal["group_by_key"] timespan = goal["timespan"] maxmin = goal["maxmin"] - evaluation_type = goal["evaluation"] + evaluation_type = achievement["evaluation"] #prepare select_cols=[func.sum(t_values.c.value).label("value"), @@ -1152,13 +1188,13 @@ def generate_statement_cache(): if evaluation_type!="immediately": - if evaluation_type=="daily": + if evaluation_type=="day": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("day","users.timezone"))) - elif evaluation_type=="weekly": + elif evaluation_type=="week": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("week","users.timezone"))) - elif evaluation_type=="monthly": + elif evaluation_type=="month": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("month","users.timezone"))) - elif evaluation_type=="yearly": + elif evaluation_type=="year": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("year","users.timezone"))) if datetime_col is not None or group_by_key is not None: @@ -1188,19 +1224,19 @@ def generate_statement_cache(): return DBSession.execute(q, {'user_id' : user_id}) @classmethod - def evaluate(cls, goal, user_id, level, goal_eval_cache_before=False, execute_triggers=True): + def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval_cache_before=False, execute_triggers=True): """evaluate the goal for the user_ids and the level""" operator = goal["operator"] - users_progress = Goal.compute_progress(goal,user_id) + users_progress = Goal.compute_progress(goal, achievement, user_id) goal_evaluation = {e["user_id"] : e["value"] for e in users_progress} goal_achieved = False if goal_eval_cache_before is False: - goal_eval_cache_before = cls.get_goal_eval_cache(goal["id"], user_id) + goal_eval_cache_before = cls.get_goal_eval_cache(goal["id"], achievement_date, user_id) new = goal_evaluation.get(user_id,0.0) @@ -1229,13 +1265,18 @@ def evaluate(cls, goal, user_id, level, goal_eval_cache_before=False, execute_tr return Goal.set_goal_eval_cache(goal=goal, user_id=user_id, + achievement_date=achievement_date, value=new, achieved = goal_achieved) else: - return Goal.get_goal_eval_cache(goal["id"], user_id) + return Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) @classmethod def select_and_execute_triggers(cls, goal, user_id, level, current_goal, value, previous_goal): + + if previous_goal == current_goal: + previous_goal = 0.0 + j = t_goal_trigger_step_executions.join(t_goal_trigger_steps) executions = {r["goal_trigger_id"] : r["step"] for r in DBSession.execute( @@ -1297,16 +1338,16 @@ def select_and_execute_triggers(cls, goal, user_id, level, current_goal, value, @classmethod - def get_goal_eval_cache(cls,goal_id,user_id): + def get_goal_eval_cache(cls,goal_id,achievement_date,user_id): """lookup and return cache entry, else return None""" - v = cache_goal_evaluation.get("%s_%s" % (goal_id,user_id)) + v = cache_goal_evaluation.get("%s_%s_%s" % (goal_id,achievement_date,user_id)) if v: return v else: return None @classmethod - def set_goal_eval_cache(cls,goal,user_id,value,achieved): + def set_goal_eval_cache(cls,goal, achievement_date, user_id,value,achieved): """set cache entry after evaluation""" cache_query = t_goal_evaluation_cache.select().where(and_(t_goal_evaluation_cache.c.goal_id==goal["id"], @@ -1342,7 +1383,7 @@ def set_goal_eval_cache(cls,goal,user_id,value,achieved): achievement_id = goal["achievement_id"] achievement = Achievement.get_achievement(achievement_id) - level = min((Achievement.get_level_int(user_id, achievement["id"]) or 0)+1,achievement["maxlevel"]) + level = min((Achievement.get_level_int(user_id, achievement["id"], achievement_date) or 0)+1,achievement["maxlevel"]) goal_output = Goal.basic_goal_output(data,level) @@ -1351,19 +1392,23 @@ def set_goal_eval_cache(cls,goal,user_id,value,achieved): "value" : value, }) - cache_goal_evaluation.set("%s_%s" % (goal["id"],user_id),goal_output) + cache_goal_evaluation.set("%s_%s_%s" % (goal["id"],achievement_date,user_id),goal_output) return goal_output @classmethod - def clear_goal_caches(cls, user_id, goal_ids): + def clear_goal_caches(cls, user_id, goal_ids_with_achievement_date): """clear the evaluation cache for the user and gaols""" - for goal_id in goal_ids: - cache_goal_evaluation.delete("%s_%s" % (goal_id,user_id)) - update_connection().execute(t_goal_evaluation_cache.delete().where(and_(t_goal_evaluation_cache.c.user_id==user_id, - t_goal_evaluation_cache.c.goal_id.in_(goal_ids)))) + for goal_id, achievement_date in goal_ids_with_achievement_date: + cache_goal_evaluation.delete("%s_%s_%s" % (goal_id, achievement_date, user_id)) + update_connection().execute(t_goal_evaluation_cache.delete().where( + and_(t_goal_evaluation_cache.c.user_id == user_id, + t_goal_evaluation_cache.c.goal_id == goal_id, + t_goal_evaluation_cache.c.achievement_date ==achievement_date )) + ) + @classmethod - def get_leaderboard(cls, goal, user_ids): + def get_leaderboard(cls, goal, achievement, achievement_date, user_ids): """get the leaderboard for the goal and userids""" q = select([t_goal_evaluation_cache.c.user_id, t_goal_evaluation_cache.c.value])\ @@ -1381,10 +1426,10 @@ def get_leaderboard(cls, goal, user_ids): for user_id in missing_users: user = User.get_user(user_id) - user_has_level = Achievement.get_level_int(user_id, achievement["id"]) + user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) user_wants_level = min((user_has_level or 0)+1, achievement["maxlevel"]) - goal_eval = Goal.evaluate(goal, user_id, user_wants_level) + goal_eval = Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level) #rerun the query items = DBSession.execute(q).fetchall() @@ -1565,9 +1610,10 @@ def insert_trigger_step_executions_after_step_upsert(mapper,connection,target): achievement = goal.achievement for user_id in user_ids: - user_has_level = Achievement.get_level_int(user_id, achievement["id"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user_id)["timezone"], achievement["evaluation"]) + user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) user_wants_level = min((user_has_level or 0) + 1, achievement["maxlevel"]) - goal_eval = Goal.evaluate(goal, user_id, user_wants_level, None, execute_triggers=False) + goal_eval = Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level, None, execute_triggers=False) previous_goal = Goal.basic_goal_output(goal, user_wants_level - 1).get("goal_goal", 0) if previous_goal == goal_eval["goal_goal"]: diff --git a/gengine/app/views.py b/gengine/app/views.py index 1235cb8..e67caee 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -10,6 +10,7 @@ import json +import pytz from pyramid.request import Request from pyramid.response import Response from pyramid.settings import asbool @@ -134,9 +135,9 @@ def _get_progress(achievements_for_user, requesting_user): achievements = Achievement.get_achievements_by_user_for_today(achievements_for_user) - def ea(achievement): + def ea(achievement, achievement_date): try: - return Achievement.evaluate(achievements_for_user, achievement["id"]) + return Achievement.evaluate(achievements_for_user, achievement["id"], achievement_date) except FormularEvaluationException as e: return { "error": "Cannot evaluate formular: " + e.message, "id" : achievement["id"] } except Exception as e: @@ -155,15 +156,49 @@ def may_view(achievement, requesting_user): return True return False - evaluatelist = [ea(achievement) for achievement in achievements if may_view(achievement, requesting_user)] + evaluatelist = [] + now = datetime.datetime.now(pytz.timezone(achievements_for_user["timezone"])) + for achievement in achievements: + if may_view(achievement, requesting_user): + achievement_dates = set() + d = Achievement.get_datetime_for_evaluation_type( + achievements_for_user["timezone"], + achievement["evaluation"], + dt=max(achievement["created_at"],achievements_for_user["created_at"]).replace(tzinfo=pytz.utc) + ) + if d == None: + achievement_dates.add(d) + else: + while d<=now: + achievement_dates.add(d) + + if achievement["evaluation"] == "yearly": + d += datetime.timedelta(days=365) + elif achievement["evaluation"] == "monthly": + d += datetime.timedelta(days=28) + elif achievement["evaluation"] == "weekly": + d += datetime.timedelta(days=7) + elif achievement["evaluation"] == "daily": + d += datetime.timedelta(days=1) + else: + break # should not happen + + d = Achievement.get_datetime_for_evaluation_type( + achievements_for_user["timezone"], + achievement["evaluation"], + dt=d + ) + + for achievement_date in achievement_dates: + evaluatelist.append(ea(achievement, achievement_date)) ret = { - "achievements" : { - x["id"] : x for x in evaluatelist if check(x) - }, - "achievement_errors" : { - x["id"] : x for x in evaluatelist if x!=None and "error" in x - } + "achievements" : [ + x for x in evaluatelist if check(x) + ], + "achievement_errors" : [ + x for x in evaluatelist if x!=None and "error" in x + ] } return ret @@ -181,9 +216,9 @@ def get_progress(request): output = _get_progress(achievements_for_user=user, requesting_user=request.user) output = copy.deepcopy(output) - for aid in list(output["achievements"].keys()): - if "new_levels" in output["achievements"][aid]: - del output["achievements"][aid]["new_levels"] + for i in range(len(output["achievements"])): + if "new_levels" in output["achievements"][i]: + del output["achievements"][i]["new_levels"] return output @@ -218,16 +253,17 @@ def increase_value(request): output = _get_progress(achievements_for_user=user, requesting_user=request.user) output = copy.deepcopy(output) - for aid in list(output["achievements"].keys()): - if len(output["achievements"][aid]["new_levels"])>0: - if "levels" in output["achievements"][aid]: - del output["achievements"][aid]["levels"] - if "priority" in output["achievements"][aid]: - del output["achievements"][aid]["priority"] - if "goals" in output["achievements"][aid]: - del output["achievements"][aid]["goals"] + for i in range(len(output["achievements"])): + if len(output["achievements"][i]["new_levels"])>0: + if "levels" in output["achievements"][i]: + del output["achievements"][i]["levels"] + if "priority" in output["achievements"][i]: + del output["achievements"][i]["priority"] + if "goals" in output["achievements"][i]: + del output["achievements"][i]["goals"] else: - del output["achievements"][aid] + del output["achievements"][i] + return output @view_config(route_name="increase_multi_values", renderer="json", request_method="POST") @@ -264,16 +300,16 @@ def increase_multi_values(request): output = _get_progress(achievements_for_user=user, requesting_user=request.user) output = copy.deepcopy(output) - for aid in list(output["achievements"].keys()): - if len(output["achievements"][aid]["new_levels"])>0: - if "levels" in output["achievements"][aid]: - del output["achievements"][aid]["levels"] - if "priority" in output["achievements"][aid]: - del output["achievements"][aid]["priority"] - if "goals" in output["achievements"][aid]: - del output["achievements"][aid]["goals"] + for i in range(len(output["achievements"])): + if len(output["achievements"][i]["new_levels"])>0: + if "levels" in output["achievements"][i]: + del output["achievements"][i]["levels"] + if "priority" in output["achievements"][i]: + del output["achievements"][i]["priority"] + if "goals" in output["achievements"][i]: + del output["achievements"][i]["goals"] else: - del output["achievements"][aid] + del output["achievements"][i] if len(output["achievements"])>0 : ret[user_id]=output From 4e69de1c96ceb3cb99639bc685c16cda0181039e Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 5 Oct 2016 17:02:02 +0200 Subject: [PATCH 055/176] index out of range --- gengine/app/views.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gengine/app/views.py b/gengine/app/views.py index e67caee..02c5b5d 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -263,6 +263,9 @@ def increase_value(request): del output["achievements"][i]["goals"] else: del output["achievements"][i] + i -= 1 + if i==len(output["achievements"])-1: + break return output @@ -310,6 +313,9 @@ def increase_multi_values(request): del output["achievements"][i]["goals"] else: del output["achievements"][i] + i -= 1 + if i == len(output["achievements"]) - 1: + break if len(output["achievements"])>0 : ret[user_id]=output From f302d80fe2275b8ef2d6d2b7680529b92541ec56 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 5 Oct 2016 17:21:34 +0200 Subject: [PATCH 056/176] index out of range --- gengine/app/views.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/gengine/app/views.py b/gengine/app/views.py index 02c5b5d..6a83901 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -252,7 +252,7 @@ def increase_value(request): output = _get_progress(achievements_for_user=user, requesting_user=request.user) output = copy.deepcopy(output) - + to_delete = list() for i in range(len(output["achievements"])): if len(output["achievements"][i]["new_levels"])>0: if "levels" in output["achievements"][i]: @@ -262,10 +262,10 @@ def increase_value(request): if "goals" in output["achievements"][i]: del output["achievements"][i]["goals"] else: - del output["achievements"][i] - i -= 1 - if i==len(output["achievements"])-1: - break + to_delete.append(i) + + for i in sorted(to_delete,reverse=True): + del output["achievements"][i] return output @@ -302,7 +302,7 @@ def increase_multi_values(request): output = _get_progress(achievements_for_user=user, requesting_user=request.user) output = copy.deepcopy(output) - + to_delete = list() for i in range(len(output["achievements"])): if len(output["achievements"][i]["new_levels"])>0: if "levels" in output["achievements"][i]: @@ -312,10 +312,10 @@ def increase_multi_values(request): if "goals" in output["achievements"][i]: del output["achievements"][i]["goals"] else: - del output["achievements"][i] - i -= 1 - if i == len(output["achievements"]) - 1: - break + to_delete.append(i) + + for i in sorted(to_delete, reverse=True): + del output["achievements"][i] if len(output["achievements"])>0 : ret[user_id]=output From f10386f1ddc180b900834c2cdef1c805a32124d4 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 6 Oct 2016 11:08:08 +0200 Subject: [PATCH 057/176] fix the loop for evaluating repeating achievements (bug introduced yesterday) --- gengine/app/views.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gengine/app/views.py b/gengine/app/views.py index 6a83901..1c67ed9 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -161,16 +161,17 @@ def may_view(achievement, requesting_user): for achievement in achievements: if may_view(achievement, requesting_user): achievement_dates = set() - d = Achievement.get_datetime_for_evaluation_type( + d = max(achievement["created_at"],achievements_for_user["created_at"]).replace(tzinfo=pytz.utc) + dr = Achievement.get_datetime_for_evaluation_type( achievements_for_user["timezone"], achievement["evaluation"], - dt=max(achievement["created_at"],achievements_for_user["created_at"]).replace(tzinfo=pytz.utc) + dt=d ) - if d == None: + if dr == None: achievement_dates.add(d) else: while d<=now: - achievement_dates.add(d) + achievement_dates.add(dr) if achievement["evaluation"] == "yearly": d += datetime.timedelta(days=365) @@ -183,7 +184,7 @@ def may_view(achievement, requesting_user): else: break # should not happen - d = Achievement.get_datetime_for_evaluation_type( + dr = Achievement.get_datetime_for_evaluation_type( achievements_for_user["timezone"], achievement["evaluation"], dt=d From 769fafec9d848ec1806d9184501fe1e7b3a26a75 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 6 Oct 2016 12:30:23 +0200 Subject: [PATCH 058/176] introduce global leaderboards --- gengine/app/model.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 371b839..dc03870 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -124,7 +124,7 @@ def get_default_token_valid_time(): Column("max_distance", ty.Integer, nullable=True), Column('priority', ty.Integer, index=True, default=0), Column('evaluation',ty.Enum("immediately","daily","weekly","monthly","yearly","end", name="evaluation_types"), default="immediately", nullable=False), - Column('relevance',ty.Enum("friends","city","own", name="relevance_types"), default="own"), + Column('relevance',ty.Enum("global","friends","city","own", name="relevance_types"), default="own"), Column('view_permission',ty.Enum("everyone", "own", name="achievement_view_permission"), default="everyone"), Column('created_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow), ) @@ -768,6 +768,8 @@ def get_relevant_users_by_achievement_and_user(cls,achievement,user_id): pass elif achievement["relevance"]=="friends": users += [x["to_id"] for x in DBSession.execute(select([t_users_users.c.to_id,], t_users_users.c.from_id==user_id)).fetchall()] + elif achievement["relevance"] == "global": + users += [x.id for x in DBSession.execute(select([t_users.c.id,])).fetchall()] return set(users) #TODO:CACHE @@ -782,6 +784,8 @@ def get_relevant_users_by_achievement_and_user_reverse(cls,achievement,user_id): pass elif achievement["relevance"]=="friends": users += [x["from_id"] for x in DBSession.execute(select([t_users_users.c.from_id,], t_users_users.c.to_id==user_id)).fetchall()] + elif achievement["relevance"] == "global": + users += [x.id for x in DBSession.execute(select([t_users.c.id, ])).fetchall()] return set(users) @classmethod @@ -881,7 +885,7 @@ def generate(): Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level,None) goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) - if achievement["relevance"]=="friends" or achievement["relevance"]=="city": + if achievement["relevance"]=="friends" or achievement["relevance"]=="city" or achievement["relevance"]=="global": goal_eval["leaderboard"] = Goal.get_leaderboard(goal, achievement, achievement_date, user_ids) goal_eval["leaderboard_position"] = list(filter(lambda x : x["user_id"]==user_id, goal_eval["leaderboard"]))[0]["position"] From 2df378e6ecdb9941660294021016a214f1764841 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 6 Oct 2016 13:28:46 +0200 Subject: [PATCH 059/176] allow to get the user using the token --- gengine/app/views.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/gengine/app/views.py b/gengine/app/views.py index 1c67ed9..0db27d3 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -353,27 +353,32 @@ def auth_login(request): except: raise APIError(400, "invalid_json", "no valid json body") + user = request.user email = doc.get("email") password = doc.get("password") - if not email or not password: - raise APIError(400, "login.email_and_password_required", "You need to send your email and password.") + if user: + #already logged in + token = user.get_or_create_token().token + else: + if not email or not password: + raise APIError(400, "login.email_and_password_required", "You need to send your email and password.") - user = DBSession.query(AuthUser).filter_by(email=email).first() + user = DBSession.query(AuthUser).filter_by(email=email).first() - if not user or not user.verify_password(password): - raise APIError(401, "login.email_or_password_invalid", "Either the email address or the password is wrong.") + if not user or not user.verify_password(password): + raise APIError(401, "login.email_or_password_invalid", "Either the email address or the password is wrong.") - if not user.active: - raise APIError(400, "user_is_not_activated", "Your user is not activated.") + if not user.active: + raise APIError(400, "user_is_not_activated", "Your user is not activated.") - token = AuthToken.generate_token() - tokenObj = AuthToken( - user_id = user.id, - token = token - ) + token = AuthToken.generate_token() + tokenObj = AuthToken( + user_id = user.id, + token = token + ) - DBSession.add(tokenObj) + DBSession.add(tokenObj) return { "token" : token, From cdc746e1f225acfe2e1e5ab6ab3a09b045c60f43 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 6 Oct 2016 13:38:05 +0200 Subject: [PATCH 060/176] nullable column should not be part of a primary key --- gengine/app/model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index dc03870..889e011 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -213,7 +213,7 @@ def get_default_token_valid_time(): t_achievements_users = Table('achievements_users', Base.metadata, Column('user_id', ty.BigInteger, ForeignKey("users.id"), primary_key = True, index=True, nullable=False), Column('achievement_id', ty.Integer, ForeignKey("achievements.id", ondelete="CASCADE"), primary_key = True, nullable=False), - Column('achievement_date', ty.DateTime, nullable=True, primary_key=True), + Column('achievement_date', ty.DateTime, nullable=True, index=True), Column('level', ty.Integer, primary_key = True, default=1), Column('updated_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, index=True), ) @@ -929,6 +929,7 @@ def generate(): update_connection().execute(t_achievements_users.insert().values({ "user_id" : user_id, "achievement_id" : achievement["id"], + "achievement_date" : achievement_date, "level" : user_wants_level })) From dd7c9f3653146a746b99fcddf06a0684e10080c0 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 6 Oct 2016 18:14:28 +0200 Subject: [PATCH 061/176] leaderboard: return basic user output instead of just the id --- gengine/app/model.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 889e011..7ecfe1b 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -410,6 +410,13 @@ def __init__(self, *args, **kw): def get_user(cls,user_id): return DBSession.execute(t_users.select().where(t_users.c.id==user_id)).fetchone() + @classmethod + def get_users(cls, user_ids): + return { + x["id"] : x for x in + DBSession.execute(t_users.select().where(t_users.c.id.in_(user_ids))).fetchall() + } + @classmethod def get_cache_expiration_time_for_today(cls,user): """return the seconds until the day of the user ends (timezone of the user). @@ -886,8 +893,8 @@ def generate(): goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) if achievement["relevance"]=="friends" or achievement["relevance"]=="city" or achievement["relevance"]=="global": - goal_eval["leaderboard"] = Goal.get_leaderboard(goal, achievement, achievement_date, user_ids) - goal_eval["leaderboard_position"] = list(filter(lambda x : x["user_id"]==user_id, goal_eval["leaderboard"]))[0]["position"] + goal_eval["leaderboard"] = Goal.get_leaderboard(goal, achievement_date, user_ids) + goal_eval["leaderboard_position"] = list(filter(lambda x : x["user"]["id"]==user_id, goal_eval["leaderboard"]))[0]["position"] goal_evals[goal["id"]]=goal_eval if not goal_eval["achieved"]: @@ -1413,7 +1420,7 @@ def clear_goal_caches(cls, user_id, goal_ids_with_achievement_date): ) @classmethod - def get_leaderboard(cls, goal, achievement, achievement_date, user_ids): + def get_leaderboard(cls, goal, achievement_date, user_ids): """get the leaderboard for the goal and userids""" q = select([t_goal_evaluation_cache.c.user_id, t_goal_evaluation_cache.c.value])\ @@ -1423,14 +1430,14 @@ def get_leaderboard(cls, goal, achievement, achievement_date, user_ids): t_goal_evaluation_cache.c.user_id.desc()) items = DBSession.execute(q).fetchall() + users = User.get_users(user_ids) + missing_users = set(user_ids)-set([x["user_id"] for x in items]) if len(missing_users)>0: #the goal has not been evaluated for some users... achievement = Achievement.get_achievement(goal["achievement_id"]) for user_id in missing_users: - user = User.get_user(user_id) - user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) user_wants_level = min((user_has_level or 0)+1, achievement["maxlevel"]) @@ -1439,7 +1446,7 @@ def get_leaderboard(cls, goal, achievement, achievement_date, user_ids): #rerun the query items = DBSession.execute(q).fetchall() - positions = [{ "user_id" : items[i]["user_id"], + positions = [{ "user": User.basic_output(users[items[i]["user_id"]]), "value" : items[i]["value"], "position" : i} for i in range(0,len(items))] From 231a8b75be3b0f51a3a3c218b667f96d109144d6 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 7 Oct 2016 13:54:23 +0200 Subject: [PATCH 062/176] increase value json body --- gengine/app/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gengine/app/views.py b/gengine/app/views.py index 0db27d3..c334de8 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -232,7 +232,11 @@ def increase_value(request): try: value = float(request.POST["value"]) except: - raise APIError(400,"invalid_value","Invalid value provided") + try: + doc = request.json_body + value = doc["value"] + except: + raise APIError(400,"invalid_value","Invalid value provided") key = request.matchdict["key"] if "key" in request.matchdict else "" variable_name = request.matchdict["variable_name"] From 90096e7ce0e4fa083ba43323cbc4678a77fc6d75 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 7 Oct 2016 15:17:18 +0200 Subject: [PATCH 063/176] test --- gengine/app/model.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 7ecfe1b..058b6b1 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -690,6 +690,9 @@ def increase_value(cls, variable_name, user, value, key): Variable.invalidate_caches_for_variable_and_user(variable["id"],user["id"]) + update_connection().commit() + + class AchievementCategory(ABase): """A category for grouping achievement types""" @@ -1413,10 +1416,11 @@ def clear_goal_caches(cls, user_id, goal_ids_with_achievement_date): """clear the evaluation cache for the user and gaols""" for goal_id, achievement_date in goal_ids_with_achievement_date: cache_goal_evaluation.delete("%s_%s_%s" % (goal_id, achievement_date, user_id)) - update_connection().execute(t_goal_evaluation_cache.delete().where( + s = update_connection() + s.execute(t_goal_evaluation_cache.delete().where( and_(t_goal_evaluation_cache.c.user_id == user_id, t_goal_evaluation_cache.c.goal_id == goal_id, - t_goal_evaluation_cache.c.achievement_date ==achievement_date )) + t_goal_evaluation_cache.c.achievement_date == achievement_date )) ) @classmethod From 2564b46690062668e600dd7038b5c831ffb4ef33 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 7 Oct 2016 15:38:37 +0200 Subject: [PATCH 064/176] asd --- gengine/app/model.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 058b6b1..00f847b 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -690,8 +690,6 @@ def increase_value(cls, variable_name, user, value, key): Variable.invalidate_caches_for_variable_and_user(variable["id"],user["id"]) - update_connection().commit() - class AchievementCategory(ABase): """A category for grouping achievement types""" From 433430ed19cdd263cd194683e317b0692ae653a2 Mon Sep 17 00:00:00 2001 From: Marcel Date: Fri, 7 Oct 2016 16:57:28 +0200 Subject: [PATCH 065/176] None-key should be treated as empty string --- gengine/app/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/app/views.py b/gengine/app/views.py index c334de8..b9b815a 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -238,7 +238,7 @@ def increase_value(request): except: raise APIError(400,"invalid_value","Invalid value provided") - key = request.matchdict["key"] if "key" in request.matchdict else "" + key = request.matchdict["key"] if ("key" in request.matchdict and request.matchdict["key"] is not None) else "" variable_name = request.matchdict["variable_name"] user = User.get_user(user_id) From 14c73d214b60ba73eb6fdeaba4af4e285b6a9bd4 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 3 Nov 2016 15:28:23 +0100 Subject: [PATCH 066/176] at_datetime --- gengine/app/model.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 00f847b..f66c261 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -587,11 +587,18 @@ def get_variable_by_name(cls,name): return DBSession.execute(t_variables.select(t_variables.c.name==name)).fetchone() @classmethod - def get_datetime_for_tz_and_group(cls,tz,group): - """get the datetime of the current row, needed for grouping""" + def get_datetime_for_tz_and_group(cls,tz,group,at_datetime=None): + """ + get the datetime of the current row, needed for grouping + the optional parameter at_datetime can provide a timezone-aware datetime which overrides the default "now" + """ + tzobj = pytz.timezone(tz) - now = datetime.datetime.now(tzobj) - #now = now.replace(tzinfo=pytz.utc) + + if not at_datetime: + now = datetime.datetime.now(tzobj) + else: + now = at_datetime t = None if group=="year": @@ -660,19 +667,20 @@ class Value(ABase): (e.g. it counts the occurences of the "events" which the variable represents) """ @classmethod - def increase_value(cls, variable_name, user, value, key): + def increase_value(cls, variable_name, user, value, key, at_datetime=None): """increase the value of the variable for the user. In addition to the variable_name there may be an application-specific key which can be used in your :class:`.Goal` definitions + The parameter at_datetime is optional and can specify a timezone-aware datetime to define when the event happened """ user_id = user["id"] tz = user["timezone"] variable = Variable.get_variable_by_name(variable_name) - datetime = Variable.get_datetime_for_tz_and_group(tz,variable["group"]) + dt = Variable.get_datetime_for_tz_and_group(tz,variable["group"],at_datetime=at_datetime) - condition = and_(t_values.c.datetime==datetime, + condition = and_(t_values.c.datetime==dt, t_values.c.variable_id==variable["id"], t_values.c.user_id==user_id, t_values.c.key==str(key)) @@ -682,7 +690,7 @@ def increase_value(cls, variable_name, user, value, key): if current_value is not None: update_connection().execute(t_values.update(condition, values={"value":current_value+value})) else: - update_connection().execute(t_values.insert({"datetime":datetime, + update_connection().execute(t_values.insert({"datetime":dt, "variable_id":variable["id"], "user_id" : user_id, "key" : str(key), From 438f881de4febe73c63334eea8e6ce157f0143b5 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 3 Nov 2016 18:36:06 +0100 Subject: [PATCH 067/176] fix an authentication issue --- gengine/app/model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index f66c261..50978a4 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -320,7 +320,10 @@ def verify_password(self, pw): return is_valid def get_or_create_token(self): - tokenObj = DBSession.query(AuthToken).filter(AuthToken.valid_until>=datetime.datetime.utcnow()).first() + tokenObj = DBSession.query(AuthToken).filter(and_( + AuthToken.valid_until>=datetime.datetime.utcnow(), + AuthToken.user_id == self.id + )).first() if not tokenObj: token = AuthToken.generate_token() From 24147759bab582a28aba6eedfd8850ea7715a1d3 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 3 Nov 2016 18:40:30 +0100 Subject: [PATCH 068/176] flush before --- gengine/app/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gengine/app/model.py b/gengine/app/model.py index 50978a4..2d852d9 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -320,6 +320,7 @@ def verify_password(self, pw): return is_valid def get_or_create_token(self): + DBSession.flush() tokenObj = DBSession.query(AuthToken).filter(and_( AuthToken.valid_until>=datetime.datetime.utcnow(), AuthToken.user_id == self.id From c264f798332128f2e8d1026637a59300eca0ffb1 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 3 Nov 2016 18:46:47 +0100 Subject: [PATCH 069/176] fix an authentication issue --- gengine/app/model.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 2d852d9..58f5780 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -320,16 +320,15 @@ def verify_password(self, pw): return is_valid def get_or_create_token(self): - DBSession.flush() tokenObj = DBSession.query(AuthToken).filter(and_( AuthToken.valid_until>=datetime.datetime.utcnow(), - AuthToken.user_id == self.id + AuthToken.user_id == self.user_id )).first() if not tokenObj: token = AuthToken.generate_token() tokenObj = AuthToken( - user_id=self.id, + user_id=self.user_id, token=token ) From f83e9b095860d65d8b0a533f4baa6e98652935d1 Mon Sep 17 00:00:00 2001 From: Marcel Date: Thu, 3 Nov 2016 18:51:28 +0100 Subject: [PATCH 070/176] we need to get the token before returning the view --- gengine/app/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gengine/app/views.py b/gengine/app/views.py index b9b815a..085309a 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -563,11 +563,12 @@ def _get_basicauth_credentials(request): if not perm_global_access_admin_ui in permissions: return request_auth(environ, start_response) else: + token_s = user.get_or_create_token().token def start_response_with_headers(status, headers, exc_info=None): cookie = SimpleCookie() - cookie['X-Auth-Token'] = user.get_or_create_token().token + cookie['X-Auth-Token'] = token_s cookie['X-Auth-Token']['path'] = get_settings().get("urlprefix", "").rstrip("/") + "/" headers.append(('Set-Cookie', cookie['X-Auth-Token'].OutputString()),) From 314842af6eb0e9a14069f96d6e23fab92c1bea53 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 13:09:46 +0100 Subject: [PATCH 071/176] add test requirements --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index 5eda0ef..7425eaa 100644 --- a/setup.py +++ b/setup.py @@ -81,4 +81,10 @@ dependency_links=[ "https://github.com/ActiDoo/PyAPNs/archive/master.zip#egg=apns", ], + extras_require={ + 'dev': [ + 'testing.postgresql', + 'names' + ] + } ) From 4f7e39d5301995d954535fc5b58c3685a61fbd0f Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 13:10:00 +0100 Subject: [PATCH 072/176] fix JSON import --- gengine/app/model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 58f5780..7fdfc9c 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -8,6 +8,7 @@ import hashlib import pytz import sqlalchemy.types as ty +from sqlalchemy.dialects.postgresql import JSON import sys from pyramid.settings import asbool @@ -52,7 +53,7 @@ Column("country", ty.String(), nullable=True, default=None), Column("region", ty.String(), nullable=True, default=None), Column("city", ty.String(), nullable=True, default=None), - Column("additional_public_data", ty.JSON(), nullable=True, default=None), + Column("additional_public_data", JSON(), nullable=True, default=None), Column('created_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow), ) @@ -258,7 +259,7 @@ def get_default_token_valid_time(): Column('id', ty.BigInteger, primary_key = True), Column('user_id', ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), index = True, nullable=False), Column('translation_id', ty.Integer, ForeignKey("translationvariables.id", ondelete="RESTRICT"), nullable = True), - Column('params', ty.JSON, nullable=True, default={}), + Column('params', JSON(), nullable=True, default={}), Column('is_read', ty.Boolean, index=True, default=False, nullable=False), Column('created_at', ty.DateTime(), nullable=False, default=datetime.datetime.utcnow, index=True), ) From 2f3cb0fc87243ec61d2268d65966d1364160d46b Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 13:10:07 +0100 Subject: [PATCH 073/176] setup test environment --- gengine/app/tests/base.py | 56 +++++++++++++++++++ gengine/app/tests/db.py | 13 +++++ gengine/app/tests/helpers.py | 98 ++++++++++++++++++++++++++++++++++ gengine/app/tests/runner.py | 26 +++++++++ gengine/app/tests/test_auth.py | 7 +++ 5 files changed, 200 insertions(+) create mode 100644 gengine/app/tests/base.py create mode 100644 gengine/app/tests/db.py create mode 100644 gengine/app/tests/helpers.py create mode 100644 gengine/app/tests/runner.py create mode 100644 gengine/app/tests/test_auth.py diff --git a/gengine/app/tests/base.py b/gengine/app/tests/base.py new file mode 100644 index 0000000..c8822c3 --- /dev/null +++ b/gengine/app/tests/base.py @@ -0,0 +1,56 @@ +import unittest +import os +from sqlalchemy.engine import create_engine +from sqlalchemy.sql.schema import Table +from gengine.metadata import init_db +from gengine.app.tests import db as db + +class BaseDBTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + if cls is BaseDBTest: + raise unittest.SkipTest("Skip BaseTest tests, it's a base class") + super(BaseDBTest, cls).setUpClass() + + def setUp(self): + self.db = db.db() + dsn = self.db.dsn() + self.engine = create_engine( + "postgresql://%(user)s@%(host)s:%(port)s/%(database)s" % { + "user" : dsn["user"], + "host": dsn["host"], + "port": dsn["port"], + "database": dsn["database"], + } + ) + init_db(self.engine) + + from gengine.metadata import Base + + Base.metadata.drop_all(self.engine) + self.engine.execute("DROP SCHEMA IF EXISTS public CASCADE") + self.engine.execute("CREATE SCHEMA IF NOT EXISTS public") + + from alembic.config import Config + from alembic import command + + alembic_cfg = Config(attributes={ + 'engine': self.engine, + 'schema': 'public' + }) + script_location = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + 'app/alembic' + ) + alembic_cfg.set_main_option("script_location", script_location) + + from gengine.app import model + + tables = [t for name, t in model.__dict__.items() if isinstance(t, Table)] + Base.metadata.create_all(self.engine, tables=tables) + + command.stamp(alembic_cfg, "head") + + def tearDown(self): + self.db.stop() + diff --git a/gengine/app/tests/db.py b/gengine/app/tests/db.py new file mode 100644 index 0000000..5db6e48 --- /dev/null +++ b/gengine/app/tests/db.py @@ -0,0 +1,13 @@ +import testing.postgresql + +db = None + +def setupDB(): + # Generate Postgresql class which shares the generated database + global db + db = testing.postgresql.PostgresqlFactory(cache_initialized_db=True) + +def unsetupDB(): + # clear cached database at end of tests + global db + db.clear_cache() diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py new file mode 100644 index 0000000..157e9b1 --- /dev/null +++ b/gengine/app/tests/helpers.py @@ -0,0 +1,98 @@ +import names +import random + +from gengine.app.model import User +from gengine.metadata import DBSession + +default_gen_data = { + "timezone" : "Europe/Berlin", + "area" : { + "min_lat" : 51.65, + "max_lat" : 51.75, + "min_lng" : 8.70, + "max_lng" : 8.79 + }, + "country" : "DE", + "region" : "NRW", + "city" : "Paderborn", + "language" : "de" +} + +alt_gen_data = { + "timezone" : "US/Eastern", + "area" : { + "min_lat" : 40.680, + "max_lat" : 40.780, + "min_lng" : -73.89, + "max_lng" : -73.97 + } +} + +class Undefined(): + pass + +undefined = Undefined() + +def randrange_float(f1,f2): + return random.random() * abs(f1 - f2) + min(f1,f2) + +def create_user( + user_id = undefined, + lat = undefined, + lng = undefined, + country = undefined, + region = undefined, + city = undefined, + timezone = undefined, + language = undefined, + friends = [], + groups = [], + additional_public_data = undefined, + gen_data = default_gen_data + ): + + if additional_public_data is undefined: + additional_public_data = { + 'first_name' : names.get_first_name(), + 'last_name' : names.get_last_name() + } + + if user_id is undefined: + user_id = (DBSession.execute("SELECT max(id) as c FROM users").scalar() or 0) + 1 + + if lat is undefined: + lat = randrange_float(gen_data["area"]["min_lat"],gen_data["area"]["max_lat"]) + + if lng is undefined: + lng = randrange_float(gen_data["area"]["min_lng"], gen_data["area"]["max_lng"]) + + if country is undefined: + country = gen_data["country"] + + if timezone is undefined: + timezone = gen_data["timezone"] + + if region is undefined: + region = gen_data["region"] + + if city is undefined: + city = gen_data["city"] + + if language is undefined: + language = gen_data["language"] + + User.set_infos( + user_id = user_id, + lat = lat, + lng = lng, + timezone = timezone, + country = country, + region = region, + city = city, + language = language, + groups = groups, + friends = friends, + additional_public_data = additional_public_data + ) + + return User.get_user(user_id) \ No newline at end of file diff --git a/gengine/app/tests/runner.py b/gengine/app/tests/runner.py new file mode 100644 index 0000000..34ece62 --- /dev/null +++ b/gengine/app/tests/runner.py @@ -0,0 +1,26 @@ +from gengine.app.tests import db as db +from gengine.metadata import init_declarative_base, init_session +import unittest +import os +import pkgutil + +init_session() +init_declarative_base() + +__path__ = [x[0] for x in os.walk(os.path.dirname(__file__))] + +def create_test_suite(): + suite = unittest.TestSuite() + for imp, modname, _ in pkgutil.walk_packages(__path__): + mod = imp.find_module(modname).load_module(modname) + for test in unittest.defaultTestLoader.loadTestsFromModule(mod): + suite.addTests(test) + return suite + +if __name__=="__main__": + try: + db.setupDB() + testSuite = create_test_suite() + text_runner = unittest.TextTestRunner().run(testSuite) + finally: + db.unsetupDB() diff --git a/gengine/app/tests/test_auth.py b/gengine/app/tests/test_auth.py new file mode 100644 index 0000000..cf09d36 --- /dev/null +++ b/gengine/app/tests/test_auth.py @@ -0,0 +1,7 @@ +from gengine.app.tests.base import BaseDBTest +from gengine.app.tests.helpers import create_user + +class TestUserCreation(BaseDBTest): + def test_user_creation(self): + user = create_user() + self.assertTrue(user.country == "DE") From 3e743fc9fc3c4146a12ea876eb91fb6492f1c68a Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 13:28:32 +0100 Subject: [PATCH 074/176] travis --- .travis.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 71916be..3bc16ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,22 @@ language: python python: - - "2.7" - + - "3.5" + +addons: + apt: + sources: + - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main' + key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + packages: + - postgresql-9.4 + # command to install dependencies install: - - python setup.py -q install - + - pip install -r requirements.txt + - pip install -e .[develop] + # command to run tests -script: nosetests +script: python -m gengine.app.tests.runner # deploy to pypi deploy: From e2a6d868aaa21fefb4fdbb4462db7ea27a19442e Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 13:30:16 +0100 Subject: [PATCH 075/176] travis --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3bc16ca..ac586a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,15 +4,15 @@ python: addons: apt: - sources: - - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main' - key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' +# sources: +# - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main' +# key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' packages: - postgresql-9.4 # command to install dependencies install: - - pip install -r requirements.txt +# - pip install -r requirements.txt - pip install -e .[develop] # command to run tests From 836617517604daf40072a27344518f28a5dfb08e Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 13:32:49 +0100 Subject: [PATCH 076/176] travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ac586a3..cea4e30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,8 @@ addons: # command to install dependencies install: -# - pip install -r requirements.txt - - pip install -e .[develop] + - pip install -r /home/travis/build/ActiDoo/gamification-engine/requirements.txt + - pip install -e .[dev] # command to run tests script: python -m gengine.app.tests.runner From 34c7bd31d7ce2f658faf906b403b9be8a9fa6a1c Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 13:36:07 +0100 Subject: [PATCH 077/176] travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cea4e30..9bd2a6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ addons: # command to install dependencies install: - - pip install -r /home/travis/build/ActiDoo/gamification-engine/requirements.txt - pip install -e .[dev] # command to run tests From 39ac4b9382d0ca1a03c4a4df866c640d525a08b2 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 13:45:36 +0100 Subject: [PATCH 078/176] travis --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9bd2a6c..eb6dec7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - "3.5" - +sudo: false addons: apt: # sources: @@ -12,6 +12,7 @@ addons: # command to install dependencies install: + - "pip install -r requirements.txt" - pip install -e .[dev] # command to run tests From 84ce0610366edbabed0865604e6e968f71de51d8 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 13:56:27 +0100 Subject: [PATCH 079/176] requirements.txt --- MANIFEST.in | 4 +--- requirements.txt | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index d31e3fd..ec33067 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1 @@ -include *.txt *.ini *.cfg *.rst -recursive-include gengine *.ico *.png *.css *.gif *.jpg *.pt *.txt *.mak *.mako *.js *.html *.xml README DUMMY alembic alembic/versions/*.py alembic/env.py -recursive-include gengine_quickstart_template *.ini +include *.txt *.ini *.cfg *.rst *.ico *.png *.css *.gif *.jpg *.pt *.txt *.mak *.mako *.js *.html *.xml *.py README DUMMY LICENSE diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8a4cad7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,46 @@ +alembic==0.8.8 +apns==2.0.1 +argon2==0.1.10 +Chameleon==2.25 +click==6.6 +dogpile.cache==0.6.2 +Flask==0.11.1 +Flask-Admin==1.4.2 +gamification-engine==0.2.0 +itsdangerous==0.24 +Jinja2==2.8 +jsl==0.2.4 +jsonschema==2.5.1 +Mako==1.0.5 +MarkupSafe==0.23 +mock==2.0.0 +PasteDeploy==1.5.2 +pbr==1.10.0 +psycopg2==2.6.2 +Pygments==2.1.3 +pymemcache==1.4.0 +pyparsing==2.1.10 +pyramid==1.7.3 +pyramid-chameleon==0.3 +pyramid-debugtoolbar==3.0.5 +pyramid-dogpile-cache==0.0.4 +pyramid-mako==1.0.2 +pyramid-tm==1.0.1 +python-editor==1.0.1 +python-gcm==0.4 +pytz==2016.7 +raven==5.31.0 +repoze.lru==0.6 +requests==2.11.1 +six==1.10.0 +SQLAlchemy==1.1.3 +transaction==1.6.1 +translationstring==1.3 +venusian==1.0 +waitress==1.0.1 +WebOb==1.6.2 +Werkzeug==0.11.11 +WTForms==2.1 +zope.deprecation==4.2.0 +zope.interface==4.3.2 +zope.sqlalchemy==0.7.7 From 29907e6758825fe9f31b515cf7028b69c35e2a59 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 13:56:48 +0100 Subject: [PATCH 080/176] requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8a4cad7..9e87478 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ click==6.6 dogpile.cache==0.6.2 Flask==0.11.1 Flask-Admin==1.4.2 -gamification-engine==0.2.0 itsdangerous==0.24 Jinja2==2.8 jsl==0.2.4 From 9d3abad74b068f2e9ca1e2d65243c907a1d6dc55 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 14:01:52 +0100 Subject: [PATCH 081/176] travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index eb6dec7..4f30509 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ addons: # command to install dependencies install: - - "pip install -r requirements.txt" + - "pip install --upgrade -r requirements.txt" - pip install -e .[dev] # command to run tests From cae48d9e9c3b12e35388e356fd655d52da0bb0c1 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 14:15:16 +0100 Subject: [PATCH 082/176] travis --- .travis.yml | 13 +++++++------ gengine/app/tests/db.py | 7 ++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4f30509..52ce853 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,15 @@ language: python +dist: trusty python: - "3.5" sudo: false addons: - apt: -# sources: -# - sourceline: 'deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main' -# key_url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' - packages: - - postgresql-9.4 + postgresql: "9.4" +services: + - postgresql +env: + - TEST_POSTGRES=/usr/lib/postgresql/9.4/bin/postgres + - TEST_INITDB=/usr/lib/postgresql/9.4/bin/initdb # command to install dependencies install: diff --git a/gengine/app/tests/db.py b/gengine/app/tests/db.py index 5db6e48..ecdb98a 100644 --- a/gengine/app/tests/db.py +++ b/gengine/app/tests/db.py @@ -1,11 +1,16 @@ import testing.postgresql +import os db = None def setupDB(): # Generate Postgresql class which shares the generated database global db - db = testing.postgresql.PostgresqlFactory(cache_initialized_db=True) + db = testing.postgresql.PostgresqlFactory( + postgres=os.environ.get("TEST_POSTGRES",None), + initdb=os.environ.get("TEST_INITDB",None), + cache_initialized_db=True + ) def unsetupDB(): # clear cached database at end of tests From e9d69093a661cae9442216499e0c98ad54b0d393 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 14:22:07 +0100 Subject: [PATCH 083/176] travis --- .travis.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 52ce853..846aa52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,20 @@ dist: trusty python: - "3.5" sudo: false -addons: - postgresql: "9.4" -services: - - postgresql +before_install: + - sudo /etc/init.d/postgresql stop + - sudo apt-get -y remove --purge postgresql-9.1 + - sudo apt-get -y remove --purge postgresql-9.2 + - sudo apt-get -y remove --purge postgresql-9.3 + - sudo apt-get -y remove --purge postgresql-9.4 + - sudo apt-get -y autoremove env: - - TEST_POSTGRES=/usr/lib/postgresql/9.4/bin/postgres - - TEST_INITDB=/usr/lib/postgresql/9.4/bin/initdb - + - TEST_POSTGRES=/usr/lib/postgresql/9.5/bin/postgres + - TEST_INITDB=/usr/lib/postgresql/9.5/bin/initdb +apt: + packages: + - postgresql-9.5 + - postgresql-contrib-9.5 # command to install dependencies install: - "pip install --upgrade -r requirements.txt" From 547cc3dbe4d4b93def790354afb590af24c335a9 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 14:25:23 +0100 Subject: [PATCH 084/176] travis --- .travis.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 846aa52..52bcab6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,15 +4,14 @@ python: - "3.5" sudo: false before_install: - - sudo /etc/init.d/postgresql stop - - sudo apt-get -y remove --purge postgresql-9.1 - - sudo apt-get -y remove --purge postgresql-9.2 - - sudo apt-get -y remove --purge postgresql-9.3 - - sudo apt-get -y remove --purge postgresql-9.4 - - sudo apt-get -y autoremove + - /etc/init.d/postgresql stop + - apt-get -y remove --purge postgresql-9.1 + - apt-get -y remove --purge postgresql-9.2 + - apt-get -y remove --purge postgresql-9.3 + - apt-get -y remove --purge postgresql-9.4 + - apt-get -y autoremove env: - - TEST_POSTGRES=/usr/lib/postgresql/9.5/bin/postgres - - TEST_INITDB=/usr/lib/postgresql/9.5/bin/initdb + - TEST_POSTGRES=/usr/lib/postgresql/9.5/bin/postgres TEST_INITDB=/usr/lib/postgresql/9.5/bin/initdb apt: packages: - postgresql-9.5 From 12846a9e82661730e42efc73a3c87db9e96f8a53 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 14:27:05 +0100 Subject: [PATCH 085/176] travis --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 52bcab6..b813fda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,14 +2,14 @@ language: python dist: trusty python: - "3.5" -sudo: false +sudo: required before_install: - - /etc/init.d/postgresql stop - - apt-get -y remove --purge postgresql-9.1 - - apt-get -y remove --purge postgresql-9.2 - - apt-get -y remove --purge postgresql-9.3 - - apt-get -y remove --purge postgresql-9.4 - - apt-get -y autoremove + - sudo /etc/init.d/postgresql stop + - sudo apt-get -y remove --purge postgresql-9.1 + - sudo apt-get -y remove --purge postgresql-9.2 + - sudo apt-get -y remove --purge postgresql-9.3 + - sudo apt-get -y remove --purge postgresql-9.4 + - sudo apt-get -y autoremove env: - TEST_POSTGRES=/usr/lib/postgresql/9.5/bin/postgres TEST_INITDB=/usr/lib/postgresql/9.5/bin/initdb apt: From 2c3bc4a7ea6a73b52f7665364846882e58173d0c Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 14:30:35 +0100 Subject: [PATCH 086/176] travis --- .travis.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index b813fda..d7c2204 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,19 +3,13 @@ dist: trusty python: - "3.5" sudo: required -before_install: - - sudo /etc/init.d/postgresql stop - - sudo apt-get -y remove --purge postgresql-9.1 - - sudo apt-get -y remove --purge postgresql-9.2 - - sudo apt-get -y remove --purge postgresql-9.3 - - sudo apt-get -y remove --purge postgresql-9.4 - - sudo apt-get -y autoremove env: - TEST_POSTGRES=/usr/lib/postgresql/9.5/bin/postgres TEST_INITDB=/usr/lib/postgresql/9.5/bin/initdb apt: packages: - postgresql-9.5 - postgresql-contrib-9.5 + - postgis # command to install dependencies install: - "pip install --upgrade -r requirements.txt" From 7e3a3e64ff46cc49bfa1807508a6bba219d1cfaa Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 14:32:36 +0100 Subject: [PATCH 087/176] add python versions to test --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index d7c2204..e2239d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ language: python dist: trusty python: + - "2.7" + - "3.4" - "3.5" sudo: required env: From 6ec8bd9b1536db430e70c56667a91b35f73af69d Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 14:35:59 +0100 Subject: [PATCH 088/176] test fix for py27 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e2239d6..9f09ed0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: - pip install -e .[dev] # command to run tests -script: python -m gengine.app.tests.runner +script: python gengine/app/tests/runner.py # deploy to pypi deploy: From b149233f7284e8ba79701c855f3bf6135d30bd81 Mon Sep 17 00:00:00 2001 From: Marcel Date: Tue, 8 Nov 2016 14:45:07 +0100 Subject: [PATCH 089/176] compatible metaclass --- gengine/app/tests/__init__.py | 1 + gengine/base/model.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 gengine/app/tests/__init__.py diff --git a/gengine/app/tests/__init__.py b/gengine/app/tests/__init__.py new file mode 100644 index 0000000..fc80254 --- /dev/null +++ b/gengine/app/tests/__init__.py @@ -0,0 +1 @@ +pass \ No newline at end of file diff --git a/gengine/base/model.py b/gengine/base/model.py index 522ec40..9612dd0 100644 --- a/gengine/base/model.py +++ b/gengine/base/model.py @@ -4,6 +4,7 @@ from sqlalchemy.orm.exc import DetachedInstanceError from sqlalchemy.sql.expression import select from sqlalchemy.sql.functions import func +from sqlalchemy.util.compat import with_metaclass from zope.sqlalchemy.datamanager import mark_changed import gengine.metadata as meta @@ -31,7 +32,7 @@ def __getattr__(cls, item): raise AttributeError(item) -class ABase(object, metaclass=ABaseMeta): +class ABase(with_metaclass(ABaseMeta, object)): """abstract base class which introduces a nice constructor for the model classes.""" def __init__(self, *args, **kw): From ec9a6bb9c379c4df30ab20a6df8713c4b5c00a62 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 9 Nov 2016 09:06:16 +0100 Subject: [PATCH 090/176] setup redis cache for testing --- gengine/__init__.py | 5 +++- gengine/app/cache.py | 36 +++++++++++++++++++++++++ gengine/app/model.py | 6 ++--- gengine/app/tests/base.py | 3 ++- gengine/app/tests/runner.py | 22 ++++++++++++++- gengine/app/tests/test_auth.py | 1 + gengine/base/cache.py | 49 +++++++++++++++++----------------- gengine/base/model.py | 5 +--- setup.py | 1 + 9 files changed, 94 insertions(+), 34 deletions(-) create mode 100644 gengine/app/cache.py diff --git a/gengine/__init__.py b/gengine/__init__.py index 29c085b..9654a32 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -32,6 +32,9 @@ def main(global_config, **settings): init_declarative_base() init_db(engine) + from gengine.app.cache import init_caches + init_caches() + from gengine.base.monkeypatch_flaskadmin import do_monkeypatch do_monkeypatch() @@ -109,7 +112,7 @@ def has_perm(request, name): init_tenantadmin(urlprefix=urlprefix, secret=settings.get("flaskadmin_secret","fKY7kJ2xSrbPC5yieEjV")) - #date serialization + #date serialization json_renderer = JSON() def datetime_adapter(obj, request): return obj.isoformat() diff --git a/gengine/app/cache.py b/gengine/app/cache.py new file mode 100644 index 0000000..861d2db --- /dev/null +++ b/gengine/app/cache.py @@ -0,0 +1,36 @@ +from gengine.base.cache import create_cache + +caches = {} + +def init_caches(): + global cache_general + cache_general = create_cache("general") + + global cache_achievement_eval + cache_achievement_eval = create_cache("achievement_eval") + + global cache_achievements_by_user_for_today + cache_achievements_by_user_for_today = create_cache("achievements_by_user_for_today") + + global cache_achievements_users_levels + cache_achievements_users_levels = create_cache("achievements_users_levels") + + global cache_translations + cache_translations = create_cache("translations") + + # The Goal evaluation Cache is implemented as a two-level cache (persistent in db, non-persistent as dogpile) + global cache_goal_evaluation + cache_goal_evaluation = create_cache("goal_evaluation") + + global cache_goal_statements + cache_goal_statements = create_cache("goal_statements") + + +def clear_all_caches(): + cache_general.invalidate(hard=True) + cache_achievement_eval.invalidate(hard=True) + cache_achievements_by_user_for_today.invalidate(hard=True) + cache_achievements_users_levels.invalidate(hard=True) + cache_translations.invalidate(hard=True) + cache_goal_evaluation.invalidate(hard=True) + cache_goal_statements.invalidate(hard=True) diff --git a/gengine/app/model.py b/gengine/app/model.py index 7fdfc9c..48cda0d 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -17,7 +17,7 @@ from gengine.app.permissions import perm_global_increase_value from gengine.base.model import ABase, exists_by_expr, datetime_trunc, calc_distance, coords, update_connection -from gengine.base.cache import cache_general, cache_goal_evaluation, cache_achievement_eval, cache_achievements_users_levels, \ +from gengine.app.cache import cache_general, cache_goal_evaluation, cache_achievement_eval, cache_achievements_users_levels, \ cache_achievements_by_user_for_today, cache_goal_statements, cache_translations from sqlalchemy import ( Table, @@ -1742,12 +1742,12 @@ def relationship(*args,**kw): mapper(Goal, t_goals, properties={ 'name_translation' : relationship(TranslationVariable), - 'properties' : relationship(GoalGoalProperty, backref='goal'), }) mapper(GoalProperty, t_goalproperties) mapper(GoalGoalProperty, t_goals_goalproperties, properties={ 'property' : relationship(GoalProperty, backref='goals'), - 'value_translation' : relationship(TranslationVariable) + 'value_translation' : relationship(TranslationVariable), + 'goal' : relationship(Goal, backref='properties',), }) mapper(GoalEvaluationCache, t_goal_evaluation_cache,properties={ 'user' : relationship(User), diff --git a/gengine/app/tests/base.py b/gengine/app/tests/base.py index c8822c3..878f2e5 100644 --- a/gengine/app/tests/base.py +++ b/gengine/app/tests/base.py @@ -5,7 +5,9 @@ from gengine.metadata import init_db from gengine.app.tests import db as db + class BaseDBTest(unittest.TestCase): + @classmethod def setUpClass(cls): if cls is BaseDBTest: @@ -53,4 +55,3 @@ def setUp(self): def tearDown(self): self.db.stop() - diff --git a/gengine/app/tests/runner.py b/gengine/app/tests/runner.py index 34ece62..686c860 100644 --- a/gengine/app/tests/runner.py +++ b/gengine/app/tests/runner.py @@ -3,6 +3,10 @@ import unittest import os import pkgutil +import testing.redis +import logging + +log = logging.getLogger(__name__) init_session() init_declarative_base() @@ -19,8 +23,24 @@ def create_test_suite(): if __name__=="__main__": try: + redis = testing.redis.RedisServer() + + from gengine.base.cache import setup_redis_cache + dsn = redis.dsn() + setup_redis_cache(dsn["host"], dsn["port"], dsn["db"]) + + from gengine.app.cache import init_caches + init_caches() + db.setupDB() testSuite = create_test_suite() text_runner = unittest.TextTestRunner().run(testSuite) finally: - db.unsetupDB() + try: + db.unsetupDB() + except: + log.exception() + try: + redis.stop() + except: + log.exception() diff --git a/gengine/app/tests/test_auth.py b/gengine/app/tests/test_auth.py index cf09d36..47e0ded 100644 --- a/gengine/app/tests/test_auth.py +++ b/gengine/app/tests/test_auth.py @@ -1,6 +1,7 @@ from gengine.app.tests.base import BaseDBTest from gengine.app.tests.helpers import create_user + class TestUserCreation(BaseDBTest): def test_user_creation(self): user = create_user() diff --git a/gengine/base/cache.py b/gengine/base/cache.py index f19b326..a9e215a 100644 --- a/gengine/base/cache.py +++ b/gengine/base/cache.py @@ -2,6 +2,20 @@ from dogpile.cache import make_region from pyramid_dogpile_cache import get_region +force_redis = None + +def setup_redis_cache(host,port,db): + """ This is used to override all caching settings in the ini file. Needed for Testing. """ + global force_redis + force_redis = { + 'host': host, + 'port': port, + 'db': db, + 'redis_expiration_time': 60 * 60 * 2, # 2 hours + 'distributed_lock': True + } + + def my_key_mangler(prefix): def s(o): if type(o) == dict: @@ -24,31 +38,18 @@ def generate_key(key): def create_cache(name): ch = None - try: - ch = get_region(name) - # The Goal evaluation Cache is implemented as a two-level cache (persistent in db, non-persistent as dogpile) - except: - ch = make_region().configure('dogpile.cache.memory') - warnings.warn("Warning: cache objects are in memory, are you creating docs?") + if force_redis: + ch = make_region().configure( + 'dogpile.cache.redis', + arguments=force_redis + ) + else: + try: + ch = get_region(name) + except: + ch = make_region().configure('dogpile.cache.memory') + warnings.warn("Warning: cache objects are in memory, are you creating docs?") ch.key_mangler = my_key_mangler(name) return ch - -cache_general = create_cache("general") -cache_achievement_eval = create_cache("achievement_eval") -cache_achievements_by_user_for_today = create_cache("achievements_by_user_for_today") -cache_achievements_users_levels = create_cache("achievements_users_levels") -cache_translations = create_cache("translations") -# The Goal evaluation Cache is implemented as a two-level cache (persistent in db, non-persistent as dogpile) -cache_goal_evaluation = create_cache("goal_evaluation") -cache_goal_statements = create_cache("goal_statements") - - -def clear_all_caches(): - cache_achievement_eval.invalidate(hard=True) - cache_achievements_by_user_for_today.invalidate(hard=True) - cache_translations.invalidate(hard=True) - cache_general.invalidate(hard=True) - cache_goal_evaluation.invalidate(hard=True) - cache_goal_statements.invalidate(hard=True) diff --git a/gengine/base/model.py b/gengine/base/model.py index 9612dd0..0ef8f4d 100644 --- a/gengine/base/model.py +++ b/gengine/base/model.py @@ -9,8 +9,6 @@ import gengine.metadata as meta -from gengine.base.cache import cache_general - class ABaseMeta(type): def __init__(cls, name, bases, nmspc): super(ABaseMeta, cls).__init__(name, bases, nmspc) @@ -98,7 +96,6 @@ def exists_by_expr(t, expr): return False -@cache_general.cache_on_arguments() def datetime_trunc(field, timezone): return "date_trunc('%(field)s', CAST(to_char(NOW() AT TIME ZONE %(timezone)s, 'YYYY-MM-DD HH24:MI:SS') AS TIMESTAMP)) AT TIME ZONE %(timezone)s" % { "field": field, @@ -106,7 +103,6 @@ def datetime_trunc(field, timezone): } -@cache_general.cache_on_arguments() def valid_timezone(timezone): try: pytz.timezone(timezone) @@ -114,6 +110,7 @@ def valid_timezone(timezone): return False return True + def update_connection(): session = meta.DBSession() mark_changed(session) diff --git a/setup.py b/setup.py index 7425eaa..19eff8c 100644 --- a/setup.py +++ b/setup.py @@ -84,6 +84,7 @@ extras_require={ 'dev': [ 'testing.postgresql', + 'testing.redis', 'names' ] } From 458b8e87216123a1292ca68103bc562c081ba144 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 9 Nov 2016 09:13:38 +0100 Subject: [PATCH 091/176] fix imports --- gengine/app/cache.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gengine/app/cache.py b/gengine/app/cache.py index 861d2db..d117484 100644 --- a/gengine/app/cache.py +++ b/gengine/app/cache.py @@ -2,6 +2,14 @@ caches = {} +cache_general = None +cache_goal_evaluation = None +cache_achievement_eval = None +cache_achievements_users_levels = None +cache_achievements_by_user_for_today = None +cache_goal_statements = None +cache_translations = None + def init_caches(): global cache_general cache_general = create_cache("general") From 5d2af9944cb121d772c642cc9bd47744a02aaf7d Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 9 Nov 2016 09:51:17 +0100 Subject: [PATCH 092/176] coverage --- .coveragerc | 2 ++ .travis.yml | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..734d668 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +source = gengine diff --git a/.travis.yml b/.travis.yml index 9f09ed0..0bb1868 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,11 +14,15 @@ apt: - postgis # command to install dependencies install: + - pip install coveralls - "pip install --upgrade -r requirements.txt" - pip install -e .[dev] # command to run tests -script: python gengine/app/tests/runner.py +script: coverage run --source=gengine gengine/app/tests/runner.py + +after_success: + coveralls # deploy to pypi deploy: From a7ed792d7e51d372e48d8efaa7e93822f6668aa1 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 9 Nov 2016 10:00:35 +0100 Subject: [PATCH 093/176] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9d50eb7..108bc71 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # gamification-engine [![Build Status](https://travis-ci.org/ActiDoo/gamification-engine.svg?branch=master)](https://travis-ci.org/ActiDoo/gamification-engine) +[![Coverage Status](https://coveralls.io/repos/github/ActiDoo/gamification-engine/badge.svg?branch=develop)](https://coveralls.io/github/ActiDoo/gamification-engine?branch=develop) [![Requirements Status](https://requires.io/github/ActiDoo/gamification-engine/requirements.svg?branch=master)](https://requires.io/github/ActiDoo/gamification-engine/requirements/?branch=master) [![Heroku](https://heroku-badge.herokuapp.com/?app=gamification-engine&root=admin)](https://gamification-engine.herokuapp.com) [![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](https://readthedocs.org/projects/gamification-engine/?badge=latest) From acd232dcdfafe6de2a33ff33b267c309733de118 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 9 Nov 2016 10:01:33 +0100 Subject: [PATCH 094/176] user test --- gengine/app/tests/helpers.py | 13 +++++++++++-- gengine/app/tests/test_auth.py | 30 +++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index 157e9b1..890e670 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -1,7 +1,7 @@ import names import random -from gengine.app.model import User +from gengine.app.model import User, Language from gengine.metadata import DBSession default_gen_data = { @@ -95,4 +95,13 @@ def create_user( additional_public_data = additional_public_data ) - return User.get_user(user_id) \ No newline at end of file + return User.get_user(user_id) + +def get_or_create_language(name): + lang = DBSession.query(Language).filter_by(name=name).first() + if not lang: + lang = Language() + lang.name = name + DBSession.add(lang) + DBSession.flush() + return lang \ No newline at end of file diff --git a/gengine/app/tests/test_auth.py b/gengine/app/tests/test_auth.py index 47e0ded..788b02a 100644 --- a/gengine/app/tests/test_auth.py +++ b/gengine/app/tests/test_auth.py @@ -1,8 +1,32 @@ from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user +from gengine.app.tests.helpers import create_user, get_or_create_language class TestUserCreation(BaseDBTest): def test_user_creation(self): - user = create_user() - self.assertTrue(user.country == "DE") + lang = get_or_create_language("en") + + user = create_user( + lat = 12.1, + lng = 12.2, + country = "RO", + region = "Transylvania", + city = "Cluj-Napoca", + timezone = "Europe/Bukarest", + language = "en", + additional_public_data = { + "first_name" : "Rudolf", + "last_name" : "Red Nose" + } + ) + + self.assertTrue(user.lat == 12.1) + self.assertTrue(user.lng == 12.2) + self.assertTrue(user.country == "RO") + self.assertTrue(user.region == "Transylvania") + self.assertTrue(user.city == "Cluj-Napoca") + self.assertTrue(user.timezone == "Europe/Bukarest") + self.assertTrue(user.language_id == lang.id) + self.assertTrue(user.additional_public_data["first_name"] == "Rudolf") + self.assertTrue(user.additional_public_data["last_name"] == "Red Nose") + From a727aa3c31e9c6e6b7b1212ae516057f6b516f98 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 9 Nov 2016 10:01:44 +0100 Subject: [PATCH 095/176] language relationship --- gengine/app/model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 48cda0d..7d1df34 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1694,7 +1694,8 @@ def relationship(*args,**kw): mapper(User, t_users, properties={ 'friends': relationship(User, secondary=t_users_users, primaryjoin=t_users.c.id==t_users_users.c.from_id, - secondaryjoin=t_users.c.id==t_users_users.c.to_id) + secondaryjoin=t_users.c.id==t_users_users.c.to_id), + 'language' : relationship(Language,backref="users"), }) mapper(UserDevice, t_user_device, properties={ From a49d981a7085c5b2bfe652d5e6b0d49059e12e2a Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 9 Nov 2016 14:16:50 +0100 Subject: [PATCH 096/176] init cache --- gengine/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/gengine/__init__.py b/gengine/__init__.py index 9654a32..eac577f 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -25,6 +25,10 @@ def main(global_config, **settings): set_settings(settings) engine = engine_from_config(settings, 'sqlalchemy.', connect_args={"options": "-c timezone=utc"}, ) + config = Configurator(settings=settings) + + from gengine.app.cache import init_caches + init_caches() from gengine.metadata import init_session, init_declarative_base, init_db @@ -32,14 +36,9 @@ def main(global_config, **settings): init_declarative_base() init_db(engine) - from gengine.app.cache import init_caches - init_caches() - from gengine.base.monkeypatch_flaskadmin import do_monkeypatch do_monkeypatch() - config = Configurator(settings=settings) - def reset_context_on_new_request(event): reset_context() config.add_subscriber(reset_context_on_new_request,NewRequest) From c48bac8b27f13d3f273e6badaa39834e266e0813 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 9 Nov 2016 14:19:10 +0100 Subject: [PATCH 097/176] init cache --- gengine/maintenance/scripts/initializedb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index 591fde0..2c0a562 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -13,6 +13,7 @@ from sqlalchemy import engine_from_config from sqlalchemy.sql.schema import Table +from gengine.app.cache import init_caches from gengine.app.permissions import perm_global_delete_user, perm_global_increase_value, perm_global_update_user_infos, \ perm_global_access_admin_ui, perm_global_read_messages, perm_global_register_device from gengine.base.model import exists_by_expr @@ -54,6 +55,7 @@ def initialize(settings,options): init_declarative_base, init_db ) + init_caches() init_session() init_declarative_base() init_db(engine) From 18cedbe1627e018559cb673b51282b633f324bf4 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 9 Nov 2016 14:20:56 +0100 Subject: [PATCH 098/176] move testing reqs --- setup.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 19eff8c..a16601d 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,11 @@ 'pyparsing', 'argon2', 'apns', - 'python-gcm' + 'python-gcm', + #Testing: + 'testing.postgresql', + 'testing.redis', + 'names' ] version = '' @@ -81,11 +85,4 @@ dependency_links=[ "https://github.com/ActiDoo/PyAPNs/archive/master.zip#egg=apns", ], - extras_require={ - 'dev': [ - 'testing.postgresql', - 'testing.redis', - 'names' - ] - } ) From 7074b6ff230a2d958169b5bca5a308125199918c Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 9 Nov 2016 14:21:13 +0100 Subject: [PATCH 099/176] move testing reqs --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0bb1868..3bf56d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ apt: install: - pip install coveralls - "pip install --upgrade -r requirements.txt" - - pip install -e .[dev] + - pip install -e . # command to run tests script: coverage run --source=gengine gengine/app/tests/runner.py From 58f33485891befb27174d017cf12cd258852239d Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 9 Nov 2016 14:32:42 +0100 Subject: [PATCH 100/176] delete devices with same push id --- gengine/app/model.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 7d1df34..a7c0eb6 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -365,11 +365,15 @@ def __unicode__(self, *args, **kwargs): @classmethod def add_or_update_device(cls, user_id, device_id, push_id, device_os, app_version): + update_connection().execute(t_user_device.delete().where(and_( + t_user_device.c.push_id == push_id, + t_user_device.c.device_os == device_os + ))) + device = DBSession.execute(t_user_device.select().where(and_( t_user_device.c.device_id == device_id, t_user_device.c.user_id == user_id ))).fetchone() - if device and (device["push_id"] != push_id or device["device_os"] != device_os or device["app_version"] != app_version From 9dbfc0f62ace1e28354928669c968b18cf3bd186 Mon Sep 17 00:00:00 2001 From: Nilakshi Date: Tue, 13 Dec 2016 10:30:43 +0100 Subject: [PATCH 101/176] Adding test cases for user deletion, updation and creation --- gengine/app/tests/helpers.py | 47 +++++++++++++++++++++++++++-- gengine/app/tests/test_auth.py | 54 +++++++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index 890e670..f8f7422 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -1,7 +1,7 @@ import names import random -from gengine.app.model import User, Language +from gengine.app.model import User, Language, t_users from gengine.metadata import DBSession default_gen_data = { @@ -50,7 +50,7 @@ def create_user( additional_public_data = undefined, gen_data = default_gen_data ): - + print(country) if additional_public_data is undefined: additional_public_data = { 'first_name' : names.get_first_name(), @@ -81,6 +81,38 @@ def create_user( if language is undefined: language = gen_data["language"] + User.set_infos( + user_id = user_id, + lat = lat, + lng = lng, + timezone = timezone, + country = country, + region = region, + city = city, + language = language, + groups = groups, + friends = friends, + additional_public_data = additional_public_data + ) + user = DBSession.execute("SELECT country FROM users WHERE id = 1") + + return User.get_user(user_id) + +def update_user( + user_id = undefined, + lat = undefined, + lng = undefined, + country = undefined, + region = undefined, + city = undefined, + timezone = undefined, + language = undefined, + friends = [], + groups = [], + additional_public_data = undefined, + gen_data = default_gen_data + ): + User.set_infos( user_id = user_id, lat = lat, @@ -97,6 +129,17 @@ def create_user( return User.get_user(user_id) +def delete_user( + user_id = undefined, + ): + + User.delete_user(user_id) + + return User.get_user(user_id) + #return DBSession.query(User).count() + + + def get_or_create_language(name): lang = DBSession.query(Language).filter_by(name=name).first() if not lang: diff --git a/gengine/app/tests/test_auth.py b/gengine/app/tests/test_auth.py index 788b02a..af528d4 100644 --- a/gengine/app/tests/test_auth.py +++ b/gengine/app/tests/test_auth.py @@ -1,9 +1,12 @@ from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, get_or_create_language +from gengine.app.tests.helpers import create_user, update_user, delete_user, get_or_create_language +from gengine.metadata import DBSession +from gengine.app.model import User class TestUserCreation(BaseDBTest): def test_user_creation(self): + return lang = get_or_create_language("en") user = create_user( @@ -30,3 +33,52 @@ def test_user_creation(self): self.assertTrue(user.additional_public_data["first_name"] == "Rudolf") self.assertTrue(user.additional_public_data["last_name"] == "Red Nose") + def test_user_updation(self): + return + lang = get_or_create_language("en") + user = create_user() + user = update_user( + user_id = user.id, + lat = 14.2, + lng = 16.3, + country = "EN", + region = "Transylvania", + city = "Cluj-Napoca", + timezone = "Europe/Bukarest", + language = "en", + additional_public_data = { + "first_name" : "Rudolf", + "last_name" : "Red Nose" + } + ) + + self.assertTrue(user.lat == 14.2) + self.assertTrue(user.lng == 16.3) + self.assertTrue(user.country == "EN") + self.assertTrue(user.region == "Transylvania") + self.assertTrue(user.city == "Cluj-Napoca") + self.assertTrue(user.timezone == "Europe/Bukarest") + self.assertTrue(user.language_id == lang.id) + self.assertTrue(user.additional_public_data["first_name"] == "Rudolf") + self.assertTrue(user.additional_public_data["last_name"] == "Red Nose") + + def test_user_deletion(self): + lang = get_or_create_language("en") + user = create_user() + + total_rows = DBSession.query(User).count() + + user = delete_user( + user_id = user.id, + ) + #diff = total_rows - total_rows_after_delete + + #self.assertTrue(diff == 1) + self.assertEqual(user, None) + + + + + + + \ No newline at end of file From 5d053bda8909f645ad021202a4440ea910b98837 Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Tue, 13 Dec 2016 18:49:25 +0100 Subject: [PATCH 102/176] test cases --- gengine/app/tests/helpers.py | 93 +++++++++++++++++++++-- gengine/app/tests/test_auth.py | 42 ++++++++--- gengine/app/tests/test_device.py | 114 +++++++++++++++++++++++++++++ gengine/app/tests/test_variable.py | 0 4 files changed, 235 insertions(+), 14 deletions(-) create mode 100644 gengine/app/tests/test_device.py create mode 100644 gengine/app/tests/test_variable.py diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index f8f7422..d84a2ac 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -1,9 +1,12 @@ import names import random -from gengine.app.model import User, Language, t_users +from gengine.app.model import User, Language from gengine.metadata import DBSession +from gengine.app.model import UserDevice, t_user_device +from sqlalchemy import (and_) + default_gen_data = { "timezone" : "Europe/Berlin", "area" : { @@ -28,6 +31,13 @@ } } +default_device_data = { + "device_id" : "1234", + "device_os" : "iOS 5", + "app_version" : "1.1", + "push_id" : "5678" +} + class Undefined(): pass @@ -110,7 +120,6 @@ def update_user( friends = [], groups = [], additional_public_data = undefined, - gen_data = default_gen_data ): User.set_infos( @@ -136,8 +145,6 @@ def delete_user( User.delete_user(user_id) return User.get_user(user_id) - #return DBSession.query(User).count() - def get_or_create_language(name): @@ -147,4 +154,80 @@ def get_or_create_language(name): lang.name = name DBSession.add(lang) DBSession.flush() - return lang \ No newline at end of file + return lang + +def create_device( + user_id=undefined, + device_id=undefined, + device_os=undefined, + push_id=undefined, + app_version=undefined, + gen_data=default_device_data + ): + + if push_id is undefined: + push_id = gen_data["push_id"] + + if device_os is undefined: + device_os = gen_data["device_os"] + + if app_version is undefined: + app_version = gen_data["app_version"] + + if device_id is undefined: + device_id = gen_data["device_id"] + + UserDevice.add_or_update_device( + device_id = device_id, + user_id = user_id, + device_os = device_os, + push_id = push_id, + app_version = app_version + ) + + device = DBSession.execute(t_user_device.select().where(and_( + t_user_device.c.device_id == device_id, + t_user_device.c.user_id == user_id + ))).fetchone() + + return device + + +def update_device( + user_id=undefined, + device_id=undefined, + device_os=undefined, + push_id=undefined, + app_version=undefined, + ): + UserDevice.add_or_update_device( + device_id=device_id, + user_id=user_id, + device_os=device_os, + push_id=push_id, + app_version=app_version + ) + + device = DBSession.execute(t_user_device.select().where(and_( + t_user_device.c.device_id == device_id, + t_user_device.c.user_id == user_id + ))).fetchone() + + return device + + + + + + + + + + + + + + + + + diff --git a/gengine/app/tests/test_auth.py b/gengine/app/tests/test_auth.py index af528d4..fd7ec37 100644 --- a/gengine/app/tests/test_auth.py +++ b/gengine/app/tests/test_auth.py @@ -1,7 +1,7 @@ from gengine.app.tests.base import BaseDBTest from gengine.app.tests.helpers import create_user, update_user, delete_user, get_or_create_language from gengine.metadata import DBSession -from gengine.app.model import User +from gengine.app.model import User, AuthUser class TestUserCreation(BaseDBTest): @@ -63,22 +63,46 @@ def test_user_updation(self): self.assertTrue(user.additional_public_data["last_name"] == "Red Nose") def test_user_deletion(self): + return lang = get_or_create_language("en") user = create_user() - total_rows = DBSession.query(User).count() - user = delete_user( - user_id = user.id, + user_id = user.id ) - #diff = total_rows - total_rows_after_delete - - #self.assertTrue(diff == 1) - self.assertEqual(user, None) + self.assertE(user, None) + + def test_verify_password(self): + return + auth_user = AuthUser() + auth_user.password = "test12345" + auth_user.active = True + auth_user.email = "test@actidoo.com" + DBSession.add(auth_user) + + iscorrect = auth_user.verify_password("test12345") + print(isCorrect) + + self.assertEqual(iscorrect, True) + + def test_create_token(self): + return + user = create_user() + auth_user = AuthUser() + auth_user.user_id = user.id + auth_user.password = "test12345" + auth_user.active = True + auth_user.email = "test@actidoo.com" + DBSession.add(auth_user) + + if auth_user.verify_password("test12345"): + token = auth_user.get_or_create_token() + + print(token) + self.assertNotEqual(token, None) - \ No newline at end of file diff --git a/gengine/app/tests/test_device.py b/gengine/app/tests/test_device.py new file mode 100644 index 0000000..299d268 --- /dev/null +++ b/gengine/app/tests/test_device.py @@ -0,0 +1,114 @@ +from gengine.app.tests.base import BaseDBTest +from gengine.app.tests.helpers import create_user, create_device, update_device +from gengine.metadata import DBSession +from gengine.app.model import Variable, Value, t_values +from sqlalchemy import (and_, select) + + +class TestUserDevice(BaseDBTest): + def test_create_user_device(self): + return + user = create_user() + + device = create_device( + device_id='3424', + user_id=user.id, + device_os='Android', + push_id='1234', + app_version='1.1' + ) + + self.assertTrue(device.device_id == '3424') + self.assertTrue(device.user_id == user.id) + self.assertTrue(device.device_os == 'Android') + self.assertTrue(device.push_id == '1234') + self.assertTrue(device.app_version == '1.1') + + def test_update_user_device(self): + return + user = create_user() + create_device(user_id=user.id) + + device = update_device( + user_id=user.id, + device_id='1256', + push_id='5126', + device_os='iOS', + app_version='1.2' + ) + + self.assertTrue(device.device_id == '1256') + self.assertTrue(device.user_id == user.id) + self.assertTrue(device.device_os == 'iOS') + self.assertTrue(device.push_id == '5126') + self.assertTrue(device.app_version == '1.2') + + def test_map_variables_to_rules(self): + return + map1 = Variable.map_variables_to_rules() + print(map1) + self.assertNotEqual(map1, None) + + def test_invalidate_caches_for_variable_and_user(self): + return + variable_id = 1, + user_id = 1 + Variable.invalidate_caches_for_variable_and_user(variable_id, user_id) + + goalsandachievements = Variable.map_variables_to_rules().get(variable_id,[]) + + self.assertEqual(goalsandachievements, None) + + def test_increase_value(self): + variable_name = "participate" + user = create_user() + + variable = Variable() + variable.id = 1 + variable.name = "participate" + DBSession.add(variable) + + value = 2 + key = 5 + + tz = user["timezone"] + variable = Variable.get_variable_by_name(variable_name) + print(variable["id"]) + + condition = and_( + t_values.c.variable_id == variable["id"], + t_values.c.user_id == user.id, + t_values.c.key == str(key)) + + current_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() + print(current_value) + + Value.increase_value(variable_name, user, value, key, at_datetime=None) + + new_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() + print(new_value) + + self.assertNotEqual(current_value, new_value) + + + + + + + + + + + + + + + + + + + + + + + diff --git a/gengine/app/tests/test_variable.py b/gengine/app/tests/test_variable.py new file mode 100644 index 0000000..e69de29 From 0e2f84144404007c6212f78913841506db014d5b Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Fri, 16 Dec 2016 17:59:39 +0100 Subject: [PATCH 103/176] added test cases for value and achievement --- gengine/app/model.py | 12 +++++-- gengine/app/tests/test_device.py | 61 ++++++++++++++++++++------------ 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index a7c0eb6..f2b0047 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -600,9 +600,9 @@ def get_datetime_for_tz_and_group(cls,tz,group,at_datetime=None): get the datetime of the current row, needed for grouping the optional parameter at_datetime can provide a timezone-aware datetime which overrides the default "now" """ - tzobj = pytz.timezone(tz) + if not at_datetime: now = datetime.datetime.now(tzobj) else: @@ -624,6 +624,7 @@ def get_datetime_for_tz_and_group(cls,tz,group,at_datetime=None): #return datetime.datetime.max.replace return datetime.datetime(year=2000,month=1,day=1,hour=0,minute=0,second=0,microsecond=0).replace(tzinfo=pytz.utc) + return t.astimezone(tzobj) @classmethod @@ -674,6 +675,7 @@ class Value(ABase): (e.g. it counts the occurences of the "events" which the variable represents) """ + @classmethod def increase_value(cls, variable_name, user, value, key, at_datetime=None): """increase the value of the variable for the user. @@ -705,7 +707,8 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): "value":value})) Variable.invalidate_caches_for_variable_and_user(variable["id"],user["id"]) - + new_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() + return new_value class AchievementCategory(ABase): """A category for grouping achievement types""" @@ -739,7 +742,9 @@ def get_achievements_by_user_for_today(cls,user): def generate_achievements_by_user_for_today(): today = datetime.date.today() by_loc = {x["id"] : x["distance"] for x in cls.get_achievements_by_location(coords(user))} + print(by_loc) by_date = cls.get_achievements_by_date(today) + print(by_date) def update(arr,distance): arr["distance"]=distance @@ -760,11 +765,14 @@ def get_achievements_by_location(cls,latlng): """return achievements which are valid in that location.""" #TODO: invalidate automatically when achievement in user's range is modified distance = calc_distance(latlng, (t_achievements.c.lat, t_achievements.c.lng)).label("distance") + print(distance) q = select([t_achievements.c.id, distance])\ .where(or_(and_(t_achievements.c.lat==None,t_achievements.c.lng==None), distance < t_achievements.c.max_distance)) + print(DBSession.execute(q).fetchall()) + return [dict(x.items()) for x in DBSession.execute(q).fetchall() if len(Goal.get_goals(x['id']))>0] @classmethod diff --git a/gengine/app/tests/test_device.py b/gengine/app/tests/test_device.py index 299d268..91a941c 100644 --- a/gengine/app/tests/test_device.py +++ b/gengine/app/tests/test_device.py @@ -1,7 +1,10 @@ from gengine.app.tests.base import BaseDBTest from gengine.app.tests.helpers import create_user, create_device, update_device from gengine.metadata import DBSession -from gengine.app.model import Variable, Value, t_values +from gengine.app.model import Variable, Value, Achievement, t_variables +from gengine.base.model import ABase, exists_by_expr, datetime_trunc, calc_distance, coords, update_connection + + from sqlalchemy import (and_, select) @@ -60,36 +63,50 @@ def test_invalidate_caches_for_variable_and_user(self): self.assertEqual(goalsandachievements, None) def test_increase_value(self): - variable_name = "participate" + return user = create_user() variable = Variable() - variable.id = 1 variable.name = "participate" + variable.group = "none" DBSession.add(variable) + print(variable) + DBSession.flush() - value = 2 - key = 5 - - tz = user["timezone"] - variable = Variable.get_variable_by_name(variable_name) - print(variable["id"]) + value = Value() + value.user_id = user.id + value.variable_id = variable.id + value.value = 1 + value.key = "5" + DBSession.add(value) + DBSession.flush() - condition = and_( - t_values.c.variable_id == variable["id"], - t_values.c.user_id == user.id, - t_values.c.key == str(key)) + new_value = value.increase_value(variable.name, user, value.value, value.key) - current_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() - print(current_value) - - Value.increase_value(variable_name, user, value, key, at_datetime=None) - - new_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() - print(new_value) - - self.assertNotEqual(current_value, new_value) + # Works correctly when removed check for datetime otherwise failed + self.assertNotEqual(value.value, new_value) + def test_get_achievements_by_location(self): + user = create_user() + achievement = Achievement() + achievement.name = "invite_users" + achievement.valid_start = "2016-12-16" + achievement.valid_end = "2016-12-18" + achievement.lat = 40.983 + achievement.lng = 41.562 + achievement.max_distance = 200000 + achievement.evaluation = "immediately" + achievement.relevance = "own" + achievement.view_permission = "everyone" + DBSession.add(achievement) + DBSession.flush() + result = achievement.get_achievements_by_user_for_today(user) + print(result) + + # Works when removed goal condition from achievements_by_location and achievements_by_date + self.assertNotEqual(result, None) + + #Need to check by adding goal From f9c76a909d1c190aebc76339515eac9963b165a3 Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Wed, 21 Dec 2016 15:19:29 +0100 Subject: [PATCH 104/176] Adding test cases for achievement, variable, values, goals --- gengine/app/model.py | 8 +- gengine/app/tests/helpers.py | 74 ++++++- gengine/app/tests/test_device.py | 321 ++++++++++++++++++++++++++++--- 3 files changed, 365 insertions(+), 38 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index f2b0047..6546c05 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -18,7 +18,7 @@ from gengine.app.permissions import perm_global_increase_value from gengine.base.model import ABase, exists_by_expr, datetime_trunc, calc_distance, coords, update_connection from gengine.app.cache import cache_general, cache_goal_evaluation, cache_achievement_eval, cache_achievements_users_levels, \ - cache_achievements_by_user_for_today, cache_goal_statements, cache_translations + cache_achievements_by_user_for_today, cache_translations from sqlalchemy import ( Table, ForeignKey, @@ -1036,9 +1036,9 @@ def get_rewards(cls,achievement_id,level): t_achievements_rewards.c.achievement_id==achievement_id))\ .order_by(t_achievements_rewards.c.from_level))\ .fetchall() + #now compute the diff :-/ build_hash = lambda x,l : hashlib.md5((str(x["id"])+str(evaluate_string(x["value"], {"level":l}))+str(Translation.trs(x["value_translation_id"], {"level":l}))).encode("UTF-8")).hexdigest() - prev_hashes = {build_hash(x,level-1) for x in prev_level} #this_hashes = {build_hash(x,level) for x in this_level} @@ -1257,7 +1257,9 @@ def generate_statement_cache(): else: return q - q = cache_goal_statements.get_or_create(str(goal["id"]),generate_statement_cache) + #q = cache_goal_statements.get_or_create(str(goal["id"]),generate_statement_cache) + # TODO: Cache the statement / Make it serializable for caching in redis + q = generate_statement_cache() return DBSession.execute(q, {'user_id' : user_id}) @classmethod diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index d84a2ac..e690c10 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -1,7 +1,7 @@ import names import random -from gengine.app.model import User, Language +from gengine.app.model import User, Language, Achievement,Goal, Variable, Value from gengine.metadata import DBSession from gengine.app.model import UserDevice, t_user_device @@ -215,11 +215,73 @@ def update_device( return device - - - - - +def create_achievement(): + achievement = Achievement() + achievement.name = "invite_users" + achievement.valid_start = "2016-12-16" + achievement.valid_end = "2016-12-18" + achievement.lat = 40.983 + achievement.lng = 41.562 + achievement.max_distance = 200000 + achievement.evaluation = "immediately" + achievement.relevance = "friends" + achievement.view_permission = "everyone" + DBSession.add(achievement) + + DBSession.flush() + + achievement = achievement.get_achievement(achievement.id) + DBSession.flush() + + return achievement + + +def create_goals(): + achievement = create_achievement() + goal = Goal() + goal.condition = """{"term": {"type": "literal", "variable": "invite_users"}}""" + goal.goal = "5*level" + goal.operator = "geq" + goal.achievement_id = achievement.id + DBSession.add(goal) + DBSession.flush() + + goal = Goal() + goal.condition = """{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""" + goal.goal = "3*level" + goal.operator = "geq" + goal.achievement_id = achievement.id + DBSession.add(goal) + + DBSession.flush() + + goals = goal.get_goals(achievement.id) + return goals + +def create_variable(): + variable = Variable() + variable.name = "participate" + variable.group = "none" + DBSession.add(variable) + + DBSession.flush() + + variable = variable.get_variable_by_name(variable.name) + return variable + +def create_value(): + variable = create_variable() + user = create_user() + + value = Value() + value.user_id = user.id + value.variable_id = variable.id + value.value = 1 + value.key = "5" + DBSession.add(value) + DBSession.flush() + + return value diff --git a/gengine/app/tests/test_device.py b/gengine/app/tests/test_device.py index 91a941c..37d5a89 100644 --- a/gengine/app/tests/test_device.py +++ b/gengine/app/tests/test_device.py @@ -1,8 +1,8 @@ from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_device, update_device +from gengine.app.tests.helpers import create_user, create_device, update_device, create_achievement, create_variable, create_value, create_goals from gengine.metadata import DBSession -from gengine.app.model import Variable, Value, Achievement, t_variables -from gengine.base.model import ABase, exists_by_expr, datetime_trunc, calc_distance, coords, update_connection +from gengine.app.model import Variable, Value, Achievement, User, t_achievements_users, Reward, AchievementReward, AchievementProperty, AchievementAchievementProperty, Goal +from gengine.base.model import update_connection from sqlalchemy import (and_, select) @@ -65,40 +65,19 @@ def test_invalidate_caches_for_variable_and_user(self): def test_increase_value(self): return user = create_user() - - variable = Variable() - variable.name = "participate" - variable.group = "none" - DBSession.add(variable) - print(variable) - DBSession.flush() - - value = Value() - value.user_id = user.id - value.variable_id = variable.id - value.value = 1 - value.key = "5" - DBSession.add(value) - DBSession.flush() + variable = create_variable() + value = create_value() new_value = value.increase_value(variable.name, user, value.value, value.key) # Works correctly when removed check for datetime otherwise failed self.assertNotEqual(value.value, new_value) + # Includes get_achievement_by_location and get_achievement_by_date def test_get_achievements_by_location(self): + return user = create_user() - achievement = Achievement() - achievement.name = "invite_users" - achievement.valid_start = "2016-12-16" - achievement.valid_end = "2016-12-18" - achievement.lat = 40.983 - achievement.lng = 41.562 - achievement.max_distance = 200000 - achievement.evaluation = "immediately" - achievement.relevance = "own" - achievement.view_permission = "everyone" - DBSession.add(achievement) + achievement = create_achievement() DBSession.flush() result = achievement.get_achievements_by_user_for_today(user) print(result) @@ -108,6 +87,290 @@ def test_get_achievements_by_location(self): #Need to check by adding goal + def test_get_relevant_users_by_achievement_friends_and_user(self): + return + #Create First user + user1 = create_user() + + # Create Second user + user2 = create_user( + lat = 85.59, + lng = 65.75, + country = "DE", + region = "Niedersachsen", + city = "Osnabrück", + timezone = "Europe/Berlin", + language = "de", + additional_public_data = { + "first_name" : "Michael", + "last_name" : "Clarke" + } + ) + + # Create Third user + user3 = create_user( + lat = 12.1, + lng = 12.2, + country = "RO", + region = "Transylvania", + city = "Cluj-Napoca", + timezone = "Europe/Bukarest", + language = "en", + additional_public_data = { + "first_name" : "Rudolf", + "last_name" : "Red Nose" + }, + friends=[1, 2] + ) + + # Create Fourth user + user4 = create_user( + lat = 25.56, + lng = 15.89, + country = "AU", + region = "Sydney", + city = "New South Wales", + timezone = "Australia", + language = "en", + additional_public_data = { + "first_name" : "Steve", + "last_name" : "Waugh" + }, + friends=[3] + ) + + achievement = create_achievement() + friendsOfuser1 = achievement.get_relevant_users_by_achievement_and_user(achievement, user1.id) + friendsOfuser3 = achievement.get_relevant_users_by_achievement_and_user(achievement, user3.id) + friendsOfuser4 = achievement.get_relevant_users_by_achievement_and_user(achievement, user4.id) + + self.assertNotIn(1, friendsOfuser1) + self.assertNotIn(2, friendsOfuser1) + self.assertNotIn(4, friendsOfuser3) + self.assertNotIn(2, friendsOfuser3) + self.assertNotIn(3, friendsOfuser4) + self.assertNotIn(2, friendsOfuser4) + + # For the relevance global + achievement1 = Achievement() + achievement1.name = "invite_users" + achievement1.valid_start = "2016-12-16" + achievement1.valid_end = "2016-12-18" + achievement1.lat = 40.983 + achievement1.lng = 41.562 + achievement1.max_distance = 200000 + achievement1.evaluation = "immediately" + achievement1.relevance = "global" + achievement1.view_permission = "everyone" + DBSession.add(achievement1) + + friendsOfuser1 = achievement.get_relevant_users_by_achievement_and_user(achievement1, user3.id) + + self.assertNotIn(1, friendsOfuser1) + + def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): + return + # Create First user + user1 = create_user() + + # Create Second user + user2 = create_user( + lat=85.59, + lng=65.75, + country="DE", + region="Niedersachsen", + city="Osnabrück", + timezone="Europe/Berlin", + language="de", + additional_public_data={ + "first_name": "Michael", + "last_name": "Clarke" + }, + friends = [1] + + ) + + # Create Third user + user3 = create_user( + lat=12.1, + lng=12.2, + country="RO", + region="Transylvania", + city="Cluj-Napoca", + timezone="Europe/Bukarest", + language="en", + additional_public_data={ + "first_name": "Rudolf", + "last_name": "Red Nose" + }, + friends=[1, 2] + ) + + # Create Fourth user + user4 = create_user( + lat=25.56, + lng=15.89, + country="AU", + region="Sydney", + city="New South Wales", + timezone="Australia", + language="en", + additional_public_data={ + "first_name": "Steve", + "last_name": "Waugh" + }, + friends=[2, 3] + ) + + achievement = create_achievement() + usersForFriend1 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user1.id) + usersForFriend2 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user2.id) + usersForFriend3 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user3.id) + usersForFriend4 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user4.id) + + print(usersForFriend1) + print(usersForFriend2) + print(usersForFriend3) + print(usersForFriend4) + + self.assertNotIn(4, usersForFriend1) + self.assertNotIn(2, usersForFriend1) + self.assertNotIn(1, usersForFriend2) + self.assertNotIn(3, usersForFriend2) + self.assertNotIn(1, usersForFriend3) + self.assertNotIn(4, usersForFriend3) + self.assertNotIn(2, usersForFriend4) + self.assertNotIn(4, usersForFriend4) + + def test_get_level(self): + return + user = create_user() + + achievement = create_achievement() + + achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) + + update_connection().execute(t_achievements_users.insert().values({ + "user_id": user.id, + "achievement_id": achievement["id"], + "achievement_date": achievement_date, + "level": 1 + })) + + DBSession.flush() + + level_object = achievement.get_level(user.id, achievement["id"], achievement_date) + + # Change achievement date + achievement_monthly = Achievement() + achievement_monthly.name = "invite_users" + achievement_monthly.valid_start = "2016-12-16" + achievement_monthly.valid_end = "2016-12-18" + achievement_monthly.lat = 40.983 + achievement_monthly.lng = 41.562 + achievement_monthly.max_distance = 200000 + achievement_monthly.evaluation = "monthly" + achievement_monthly.relevance = "friends" + achievement_monthly.view_permission = "everyone" + DBSession.add(achievement_monthly) + DBSession.flush() + + achievement_date1 = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement_monthly["evaluation"]) + + level_object1 = achievement.get_level(user.id, achievement["id"], achievement_date1) + + # Test for get_level as integer + level = achievement.get_level_int(user.id, achievement.id, achievement_date) + level1 = achievement.get_level_int(user.id, achievement.id, achievement_date1) + + # Passed cases + self.assertNotEqual(level_object, None) + self.assertEqual(level_object1, []) + self.assertEqual(level, 1) + self.assertEqual(level1, 0) + + # Failed cases + self.assertEqual(level1, 1) + self.assertEqual(level_object, None) + + def test_get_rewards(self): + return + achievement = create_achievement() + + reward = Reward() + reward.name = "badge" + DBSession.add(reward) + DBSession.flush() + + achievement_reward = AchievementReward() + achievement_reward.achievement_id = achievement.id + achievement_reward.reward_id = reward.id + achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level1}.png" + achievement_reward.from_level = 2 + DBSession.add(achievement_reward) + DBSession.flush() + + achievement_reward = AchievementReward() + achievement_reward.achievement_id = achievement.id + achievement_reward.reward_id = reward.id + achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level2}.png" + achievement_reward.from_level = 3 + DBSession.add(achievement_reward) + DBSession.flush() + + rewardlist1 = Achievement.get_rewards(achievement.id, 1) + print(rewardlist1) + + rewardlist2 = Achievement.get_rewards(achievement.id, 2) + print(rewardlist2) + + rewardlist3 = Achievement.get_rewards(achievement.id, 3) + print(rewardlist3) + + self.assertNotEqual(rewardlist2, None) + self.assertNotEqual(rewardlist3, None) + self.assertEqual(rewardlist1, None) + + def test_get_achievement_properties(self): + return + achievement = create_achievement() + + achievementproperty = AchievementProperty() + achievementproperty.name = "xp" + DBSession.add(achievementproperty) + DBSession.flush() + + achievements_achievementproperty = AchievementAchievementProperty() + achievements_achievementproperty.achievement_id = achievement.id + achievements_achievementproperty.property_id = achievementproperty.id + achievements_achievementproperty.value = "5" + achievements_achievementproperty.from_level = 2 + DBSession.add(achievements_achievementproperty) + DBSession.flush() + + result1 = Achievement.get_achievement_properties(achievement.id, 4) + + result2 = Achievement.get_achievement_properties(achievement.id, 1) + + self.assertNotEqual(result1, []) + self.assertNotEqual(result2, []) + + def test_evaluate_achievement(self): + return + + def test_compute_progress(self): + goals = create_goals() + achievement = create_achievement() + user = create_user() + + result1 = Goal.compute_progress(goals[0], achievement, user.id) + result2 = Goal.compute_progress(goals[1], achievement, user.id) + + self.assertIsNotNone(result1) + self.assertIsNotNone(result2) + + + From 04b10beb09df5813a91e8c62d4d748552db431c7 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 21 Dec 2016 18:20:57 +0100 Subject: [PATCH 105/176] fix test runner --- gengine/app/tests/base.py | 9 ++++----- gengine/base/util.py | 14 ++++++++++++++ gengine/metadata.py | 27 +++++++++++++++------------ 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/gengine/app/tests/base.py b/gengine/app/tests/base.py index 878f2e5..949065f 100644 --- a/gengine/app/tests/base.py +++ b/gengine/app/tests/base.py @@ -2,9 +2,8 @@ import os from sqlalchemy.engine import create_engine from sqlalchemy.sql.schema import Table -from gengine.metadata import init_db -from gengine.app.tests import db as db - +from gengine.metadata import init_db, init_session, get_sessionmaker +from gengine.app.tests import db class BaseDBTest(unittest.TestCase): @@ -25,9 +24,9 @@ def setUp(self): "database": dsn["database"], } ) - init_db(self.engine) - + init_session(override_session=get_sessionmaker()(bind=self.engine), replace=True) from gengine.metadata import Base + Base.metadata.bind = self.engine Base.metadata.drop_all(self.engine) self.engine.execute("DROP SCHEMA IF EXISTS public CASCADE") diff --git a/gengine/base/util.py b/gengine/base/util.py index 9710609..ed5a2d1 100644 --- a/gengine/base/util.py +++ b/gengine/base/util.py @@ -10,3 +10,17 @@ def __getattr__(self, name): def __setattr__(self, key, value): super().__getattribute__("obj")[key] = value + + +class Proxy(object): + def __init__(self): + self.target = None + + def __getattr__(self, name): + return getattr(self.target, name) + + def __setattr__(self, name, value): + if name == "target": + return object.__setattr__(self, name, value) + else: + setattr(self.target, name, value) \ No newline at end of file diff --git a/gengine/metadata.py b/gengine/metadata.py index 8d5ee82..e2ba543 100644 --- a/gengine/metadata.py +++ b/gengine/metadata.py @@ -5,6 +5,9 @@ from zope.sqlalchemy.datamanager import ZopeTransactionExtension from sqlalchemy.ext.declarative.api import declarative_base +from gengine.base.util import Proxy + + class MySession(Session): """This allow us to use the flask-admin sqla extension, which uses DBSession.commit() rather than transaction.commit()""" def commit(self,*args,**kw): @@ -13,21 +16,22 @@ def commit(self,*args,**kw): def rollback(self,*args,**kw): transaction.abort(*args,**kw) -DBSession=None +DBSession=Proxy() + +def get_sessionmaker(): + return sessionmaker( + extension=ZopeTransactionExtension(), + class_=MySession + ) -def init_session(override_session=None): +def init_session(override_session=None, replace=False): global DBSession - if DBSession: + if DBSession.target and not replace: return if override_session: - DBSession = override_session + DBSession.target = override_session else: - DBSession = scoped_session( - sessionmaker( - extension=ZopeTransactionExtension(), - class_=MySession - ) - ) + DBSession.target = scoped_session(get_sessionmaker()) Base=None @@ -36,7 +40,7 @@ def init_declarative_base(override_base=None): if Base: return if override_base: - Base=override_base + Base = override_base else: convention = { "ix": 'ix_%(column_0_label)s', @@ -51,4 +55,3 @@ def init_declarative_base(override_base=None): def init_db(engine): DBSession.configure(bind=engine) Base.metadata.bind = engine - \ No newline at end of file From e05459ba4b2346c92f1ef3e885e97fb821254470 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Mon, 2 Jan 2017 17:05:36 +0100 Subject: [PATCH 106/176] proxy callable --- gengine/base/util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gengine/base/util.py b/gengine/base/util.py index ed5a2d1..ad092fb 100644 --- a/gengine/base/util.py +++ b/gengine/base/util.py @@ -23,4 +23,8 @@ def __setattr__(self, name, value): if name == "target": return object.__setattr__(self, name, value) else: - setattr(self.target, name, value) \ No newline at end of file + setattr(self.target, name, value) + + def __call__(self, *args, **kwargs): + return self.target(*args, **kwargs) + \ No newline at end of file From eab7e2dc3404f2b487dbd39771d874cc0cf7a119 Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Fri, 6 Jan 2017 11:21:38 +0100 Subject: [PATCH 107/176] Adding new test cases --- development.ini | 4 ++-- gengine/app/cache.py | 8 ++++---- gengine/app/tests/test_device.py | 8 ++++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/development.ini b/development.ini index 1f2e45c..5de6c5d 100644 --- a/development.ini +++ b/development.ini @@ -15,7 +15,7 @@ pyramid.includes = pyramid_debugtoolbar pyramid_tm -sqlalchemy.url = postgres://user:password@127.0.0.1/gengine +sqlalchemy.url = postgres://nilakshi@127.0.0.1/gengine #reverse proxy settings force_https = false @@ -32,7 +32,7 @@ flaskadmin_secret = 87ghsjkdjfhg85grsfgsdfghwez89hsuif # replace with random str # dogpile cache dogpile_cache.backend = dogpile.cache.null -dogpile_cache.general.backend = dogpile.cache.dbm +dogpile_incache.general.backend = dogpile.cache.dbm dogpile_cache.general.arguments.filename = general.dbm dogpile_cache.achievement_eval.backend = dogpile.cache.dbm diff --git a/gengine/app/cache.py b/gengine/app/cache.py index d117484..33c3734 100644 --- a/gengine/app/cache.py +++ b/gengine/app/cache.py @@ -7,7 +7,7 @@ cache_achievement_eval = None cache_achievements_users_levels = None cache_achievements_by_user_for_today = None -cache_goal_statements = None +#cache_goal_statements = None cache_translations = None def init_caches(): @@ -30,8 +30,8 @@ def init_caches(): global cache_goal_evaluation cache_goal_evaluation = create_cache("goal_evaluation") - global cache_goal_statements - cache_goal_statements = create_cache("goal_statements") + #global cache_goal_statements + #cache_goal_statements = create_memory_cache("goal_statements") def clear_all_caches(): @@ -41,4 +41,4 @@ def clear_all_caches(): cache_achievements_users_levels.invalidate(hard=True) cache_translations.invalidate(hard=True) cache_goal_evaluation.invalidate(hard=True) - cache_goal_statements.invalidate(hard=True) + #cache_goal_statements.invalidate(hard=True) diff --git a/gengine/app/tests/test_device.py b/gengine/app/tests/test_device.py index 37d5a89..d0cd45f 100644 --- a/gengine/app/tests/test_device.py +++ b/gengine/app/tests/test_device.py @@ -369,6 +369,14 @@ def test_compute_progress(self): self.assertIsNotNone(result1) self.assertIsNotNone(result2) + def test_evaluate_goal(self): + achievement = create_achievement() + goal = create_goals() + user = create_user() + achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) + + Goal.evaluate(goal, achievement, achievement_date, user.id, 2, goal_eval_cache_before=False,execute_triggers=True) + From 897944cd9bcd1378ae31b03c377079d2e62103fe Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Sat, 7 Jan 2017 07:53:03 +0100 Subject: [PATCH 108/176] Added remaining tests for Goal --- gengine/app/model.py | 16 +++---- gengine/app/tests/helpers.py | 65 ++++++++++++++++++---------- gengine/app/tests/test_device.py | 73 +++++++++++++++++++++++++++++--- gengine/app/tests/test_goal.py | 0 4 files changed, 116 insertions(+), 38 deletions(-) create mode 100644 gengine/app/tests/test_goal.py diff --git a/gengine/app/model.py b/gengine/app/model.py index 6546c05..24b5415 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -742,9 +742,7 @@ def get_achievements_by_user_for_today(cls,user): def generate_achievements_by_user_for_today(): today = datetime.date.today() by_loc = {x["id"] : x["distance"] for x in cls.get_achievements_by_location(coords(user))} - print(by_loc) by_date = cls.get_achievements_by_date(today) - print(by_date) def update(arr,distance): arr["distance"]=distance @@ -765,13 +763,11 @@ def get_achievements_by_location(cls,latlng): """return achievements which are valid in that location.""" #TODO: invalidate automatically when achievement in user's range is modified distance = calc_distance(latlng, (t_achievements.c.lat, t_achievements.c.lng)).label("distance") - print(distance) q = select([t_achievements.c.id, distance])\ .where(or_(and_(t_achievements.c.lat==None,t_achievements.c.lng==None), distance < t_achievements.c.max_distance)) - print(DBSession.execute(q).fetchall()) return [dict(x.items()) for x in DBSession.execute(q).fetchall() if len(Goal.get_goals(x['id']))>0] @@ -1233,14 +1229,12 @@ def generate_statement_cache(): q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("month","users.timezone"))) elif evaluation_type=="year": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("year","users.timezone"))) - if datetime_col is not None or group_by_key is not None: if datetime_col is not None: q = q.group_by(datetime_col) if group_by_key is not None: q = q.group_by(t_values.c.key) - query_with_groups = q.alias() select_cols2 = [query_with_groups.c.user_id] @@ -1271,6 +1265,8 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval users_progress = Goal.compute_progress(goal, achievement, user_id) goal_evaluation = {e["user_id"] : e["value"] for e in users_progress} + print("In evaluate") + print(goal_evaluation) goal_achieved = False @@ -1278,6 +1274,7 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval goal_eval_cache_before = cls.get_goal_eval_cache(goal["id"], achievement_date, user_id) new = goal_evaluation.get(user_id,0.0) + print("new", new) if goal_eval_cache_before is None or goal_eval_cache_before.get("value",0.0)!=goal_evaluation.get(user_id,0.0): @@ -1285,9 +1282,9 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval params = { "level" : level } - + print("params", params) goal_goal = evaluate_value_expression(goal["goal"], params) - + print("goal_goal",goal_goal) if goal_goal is not None and operator=="geq" and new>=goal_goal: goal_achieved = True new = min(new,goal_goal) @@ -1297,7 +1294,7 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval new = max(new,goal_goal) previous_goal = Goal.basic_goal_output(goal, level-1).get("goal_goal",0) - + print("previous_goal") # Evaluate triggers if execute_triggers: Goal.select_and_execute_triggers(goal = goal, user_id = user_id, level = level, current_goal = goal_goal, previous_goal = previous_goal, value = new) @@ -1380,6 +1377,7 @@ def select_and_execute_triggers(cls, goal, user_id, level, current_goal, value, def get_goal_eval_cache(cls,goal_id,achievement_date,user_id): """lookup and return cache entry, else return None""" v = cache_goal_evaluation.get("%s_%s_%s" % (goal_id,achievement_date,user_id)) + print(v) if v: return v else: diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index e690c10..b3bf818 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -1,7 +1,7 @@ import names import random -from gengine.app.model import User, Language, Achievement,Goal, Variable, Value +from gengine.app.model import User, Language, Achievement,Goal, Variable, Value, t_goals, GoalProperty, GoalGoalProperty, TranslationVariable, t_goals_goalproperties from gengine.metadata import DBSession from gengine.app.model import UserDevice, t_user_device @@ -60,7 +60,6 @@ def create_user( additional_public_data = undefined, gen_data = default_gen_data ): - print(country) if additional_public_data is undefined: additional_public_data = { 'first_name' : names.get_first_name(), @@ -215,6 +214,7 @@ def update_device( return device + def create_achievement(): achievement = Achievement() achievement.name = "invite_users" @@ -236,28 +236,58 @@ def create_achievement(): return achievement -def create_goals(): - achievement = create_achievement() +def create_goals(achievement): goal = Goal() goal.condition = """{"term": {"type": "literal", "variable": "invite_users"}}""" goal.goal = "5*level" - goal.operator = "geq" + goal.operator = "leq" goal.achievement_id = achievement.id DBSession.add(goal) DBSession.flush() - goal = Goal() - goal.condition = """{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""" - goal.goal = "3*level" - goal.operator = "geq" - goal.achievement_id = achievement.id - DBSession.add(goal) + goal1 = Goal() + goal1.condition = """{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""" + goal1.goal = "3*level" + goal1.group_by_key = True + goal1.operator = "geq" + goal1.achievement_id = achievement.id + DBSession.add(goal1) + DBSession.flush() + goals = DBSession.execute(t_goals.select(t_goals.c.achievement_id == achievement.id)).fetchall() DBSession.flush() + print(goals) - goals = goal.get_goals(achievement.id) return goals + +def create_goal_properties(goal_id): + + goal_property = GoalProperty() + goal_property.name = "participate" + goal_property.is_variable = True + DBSession.add(goal_property) + DBSession.flush() + + translation_variable = TranslationVariable() + translation_variable.name = "invite_users_goal_name" + DBSession.add(translation_variable) + DBSession.flush() + + goals_goal_property = GoalGoalProperty() + goals_goal_property.goal_id = goal_id + goals_goal_property.property_id = goal_property.id + goals_goal_property.value = "7" + goals_goal_property.value_translation_id = translation_variable.id + goals_goal_property.from_level = 2 + DBSession.add(goals_goal_property) + DBSession.flush() + + goals_goal_property_result = DBSession.execute(t_goals_goalproperties.select().where(t_goals_goalproperties.c.goal_id == goal_id)).fetchone() + + return goals_goal_property_result + + def create_variable(): variable = Variable() variable.name = "participate" @@ -282,14 +312,3 @@ def create_value(): DBSession.flush() return value - - - - - - - - - - - diff --git a/gengine/app/tests/test_device.py b/gengine/app/tests/test_device.py index d0cd45f..8554d5a 100644 --- a/gengine/app/tests/test_device.py +++ b/gengine/app/tests/test_device.py @@ -1,5 +1,5 @@ from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_device, update_device, create_achievement, create_variable, create_value, create_goals +from gengine.app.tests.helpers import create_user, create_device, update_device, create_achievement, create_variable, create_value, create_goals, create_goal_properties from gengine.metadata import DBSession from gengine.app.model import Variable, Value, Achievement, User, t_achievements_users, Reward, AchievementReward, AchievementProperty, AchievementAchievementProperty, Goal from gengine.base.model import update_connection @@ -359,23 +359,84 @@ def test_evaluate_achievement(self): return def test_compute_progress(self): - goals = create_goals() + return achievement = create_achievement() + goals = create_goals(achievement) user = create_user() - result1 = Goal.compute_progress(goals[0], achievement, user.id) + print(result1) result2 = Goal.compute_progress(goals[1], achievement, user.id) - + print(result2) self.assertIsNotNone(result1) self.assertIsNotNone(result2) + # If group_by_key attribute for goal is not set, then default value is considered as False and NOT None + # In compute_progress function , group_by_key is compared with None. Is it desired or need to change it to False? + def test_evaluate_goal(self): + return achievement = create_achievement() - goal = create_goals() + goal = create_goals(achievement) user = create_user() achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) - Goal.evaluate(goal, achievement, achievement_date, user.id, 2, goal_eval_cache_before=False,execute_triggers=True) + evaluation_result = Goal.evaluate(goal[0], achievement, achievement_date, user.id, 5, goal_eval_cache_before=False,execute_triggers=True) + print(evaluation_result) + evaluation_result1 = Goal.evaluate(goal[1], achievement, achievement_date, user.id, 2, goal_eval_cache_before=False,execute_triggers=True) + print(evaluation_result1) + + # True cases + self.assertLessEqual(evaluation_result["value"], 25.0) + self.assertEqual(evaluation_result["achieved"], True) + + # False cases + self.assertGreater(evaluation_result1["value"], 0.0) + self.assertEqual(evaluation_result1["achieved"], True) + + # What is "new" in function evaluate()? + + def test_execute_triggers(self): + return + # Function is called inside evaluate_goal function + achievement = create_achievement() + goal = create_goals(achievement) + user = create_user() + print(goal[0]) + level = 5 + previous_goal = Goal.basic_goal_output(goal[0], level - 1).get("goal_goal", 0) + print(previous_goal) + current_goal = Goal.basic_goal_output(goal[0], level).get("goal_goal", 0) + print(current_goal) + + # What is "value" here? + # We are considering parameter value as 0 + value = 0.0 + result = Goal.select_and_execute_triggers(goal[1], user.id, level, current_goal, value, previous_goal) + print(result) # None + # What is expected result? + + def test_get_goal_properties(self): + achievement = create_achievement() + goals = create_goals(achievement) + + create_goal_properties(goals[0].id) + + level = 4 + result = Goal.get_goal_properties(goals[0].id, level) + print(result) + + level1 = 1 + result1 = Goal.get_goal_properties(goals[0].id, level1) + print(result1) + + # True test + self.assertIsNotNone(result) + + # False test + self.assertNotEqual(result1, []) + + + diff --git a/gengine/app/tests/test_goal.py b/gengine/app/tests/test_goal.py new file mode 100644 index 0000000..e69de29 From 13727ce8487bd7581410fbc73372f83f72bc44a3 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Tue, 10 Jan 2017 12:34:14 +0100 Subject: [PATCH 109/176] fix cache clearing --- gengine/app/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/app/admin.py b/gengine/app/admin.py index fe99121..4483028 100644 --- a/gengine/app/admin.py +++ b/gengine/app/admin.py @@ -249,7 +249,7 @@ def index(self): self._template_args['clear_caches_form'] = self.clear_caches_form = ClearCacheForm(request.form) if request.method == 'POST': - from gengine.base.cache import clear_all_caches + from gengine.app.cache import clear_all_caches if self.clear_caches_form.clear_check.data: clear_all_caches() self._template_args['msgs'].append("All caches cleared!") From ab232078f97aff19d0088f817b1032b65d31ee41 Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Wed, 11 Jan 2017 18:46:44 +0100 Subject: [PATCH 110/176] Added further test cases for Goal, changes in previous test cases, code refactoring --- gengine/app/model.py | 13 +- gengine/app/tests/helpers.py | 31 +- gengine/app/tests/test_achievement1.py | 290 ++++++++++++++++++ gengine/app/tests/test_device.py | 400 +------------------------ gengine/app/tests/test_goal.py | 203 +++++++++++++ gengine/app/tests/test_value.py | 35 +++ gengine/base/model.py | 2 +- 7 files changed, 557 insertions(+), 417 deletions(-) create mode 100644 gengine/app/tests/test_achievement1.py create mode 100644 gengine/app/tests/test_value.py diff --git a/gengine/app/model.py b/gengine/app/model.py index 24b5415..7ff8eb3 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1229,11 +1229,14 @@ def generate_statement_cache(): q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("month","users.timezone"))) elif evaluation_type=="year": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("year","users.timezone"))) - if datetime_col is not None or group_by_key is not None: + + print("datetime_col ", datetime_col) + print("group_by_key ", group_by_key) + if datetime_col is not None or group_by_key is not False: if datetime_col is not None: q = q.group_by(datetime_col) - if group_by_key is not None: + if group_by_key is not False: q = q.group_by(t_values.c.key) query_with_groups = q.alias() @@ -1254,6 +1257,8 @@ def generate_statement_cache(): #q = cache_goal_statements.get_or_create(str(goal["id"]),generate_statement_cache) # TODO: Cache the statement / Make it serializable for caching in redis q = generate_statement_cache() + print("progress") + print(DBSession.execute(q, {'user_id' : user_id})) return DBSession.execute(q, {'user_id' : user_id}) @classmethod @@ -1263,10 +1268,7 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval operator = goal["operator"] users_progress = Goal.compute_progress(goal, achievement, user_id) - goal_evaluation = {e["user_id"] : e["value"] for e in users_progress} - print("In evaluate") - print(goal_evaluation) goal_achieved = False @@ -1274,7 +1276,6 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval goal_eval_cache_before = cls.get_goal_eval_cache(goal["id"], achievement_date, user_id) new = goal_evaluation.get(user_id,0.0) - print("new", new) if goal_eval_cache_before is None or goal_eval_cache_before.get("value",0.0)!=goal_evaluation.get(user_id,0.0): diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index b3bf818..e20db0f 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -248,7 +248,7 @@ def create_goals(achievement): goal1 = Goal() goal1.condition = """{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""" goal1.goal = "3*level" - goal1.group_by_key = True + goal1.group_by_key = False goal1.operator = "geq" goal1.achievement_id = achievement.id DBSession.add(goal1) @@ -288,26 +288,31 @@ def create_goal_properties(goal_id): return goals_goal_property_result -def create_variable(): +def create_variable( + variable_name = undefined, + variable_group = undefined, + ): variable = Variable() - variable.name = "participate" - variable.group = "none" + variable.name = variable_name + variable.group = variable_group DBSession.add(variable) - DBSession.flush() - variable = variable.get_variable_by_name(variable.name) return variable -def create_value(): - variable = create_variable() - user = create_user() + +def create_value( + user_id=undefined, + variable_id=undefined, + var_value=undefined, + key="", + ): value = Value() - value.user_id = user.id - value.variable_id = variable.id - value.value = 1 - value.key = "5" + value.user_id = user_id + value.variable_id = variable_id + value.value = var_value + value.key = key DBSession.add(value) DBSession.flush() diff --git a/gengine/app/tests/test_achievement1.py b/gengine/app/tests/test_achievement1.py new file mode 100644 index 0000000..f8954d4 --- /dev/null +++ b/gengine/app/tests/test_achievement1.py @@ -0,0 +1,290 @@ +from gengine.app.tests.base import BaseDBTest +from gengine.app.tests.helpers import create_user, create_achievement +from gengine.metadata import DBSession +from gengine.app.model import Achievement, User + + +class TestAchievement(BaseDBTest): + # Includes get_achievement_by_location and get_achievement_by_date + def test_get_achievements_by_location(self): + return + user = create_user() + achievement = create_achievement() + DBSession.flush() + result = achievement.get_achievements_by_user_for_today(user) + print(result) + + # Works when removed goal condition from achievements_by_location and achievements_by_date + self.assertNotEqual(result, None) + + #Need to check by adding goal + + def test_get_relevant_users_by_achievement_friends_and_user(self): + return + #Create First user + user1 = create_user() + + # Create Second user + user2 = create_user( + lat = 85.59, + lng = 65.75, + country = "DE", + region = "Niedersachsen", + city = "Osnabrück", + timezone = "Europe/Berlin", + language = "de", + additional_public_data = { + "first_name" : "Michael", + "last_name" : "Clarke" + } + ) + + # Create Third user + user3 = create_user( + lat = 12.1, + lng = 12.2, + country = "RO", + region = "Transylvania", + city = "Cluj-Napoca", + timezone = "Europe/Bukarest", + language = "en", + additional_public_data = { + "first_name" : "Rudolf", + "last_name" : "Red Nose" + }, + friends=[1, 2] + ) + + # Create Fourth user + user4 = create_user( + lat = 25.56, + lng = 15.89, + country = "AU", + region = "Sydney", + city = "New South Wales", + timezone = "Australia", + language = "en", + additional_public_data = { + "first_name" : "Steve", + "last_name" : "Waugh" + }, + friends=[3] + ) + + achievement = create_achievement() + friendsOfuser1 = achievement.get_relevant_users_by_achievement_and_user(achievement, user1.id) + friendsOfuser3 = achievement.get_relevant_users_by_achievement_and_user(achievement, user3.id) + friendsOfuser4 = achievement.get_relevant_users_by_achievement_and_user(achievement, user4.id) + + self.assertNotIn(1, friendsOfuser1) + self.assertNotIn(2, friendsOfuser1) + self.assertNotIn(4, friendsOfuser3) + self.assertNotIn(2, friendsOfuser3) + self.assertNotIn(3, friendsOfuser4) + self.assertNotIn(2, friendsOfuser4) + + # For the relevance global + achievement1 = Achievement() + achievement1.name = "invite_users" + achievement1.valid_start = "2016-12-16" + achievement1.valid_end = "2016-12-18" + achievement1.lat = 40.983 + achievement1.lng = 41.562 + achievement1.max_distance = 200000 + achievement1.evaluation = "immediately" + achievement1.relevance = "global" + achievement1.view_permission = "everyone" + DBSession.add(achievement1) + + friendsOfuser1 = achievement.get_relevant_users_by_achievement_and_user(achievement1, user3.id) + + self.assertNotIn(1, friendsOfuser1) + + def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): + return + # Create First user + user1 = create_user() + + # Create Second user + user2 = create_user( + lat=85.59, + lng=65.75, + country="DE", + region="Niedersachsen", + city="Osnabrück", + timezone="Europe/Berlin", + language="de", + additional_public_data={ + "first_name": "Michael", + "last_name": "Clarke" + }, + friends = [1] + ) + + # Create Third user + user3 = create_user( + lat=12.1, + lng=12.2, + country="RO", + region="Transylvania", + city="Cluj-Napoca", + timezone="Europe/Bukarest", + language="en", + additional_public_data={ + "first_name": "Rudolf", + "last_name": "Red Nose" + }, + friends=[1, 2] + ) + + # Create Fourth user + user4 = create_user( + lat=25.56, + lng=15.89, + country="AU", + region="Sydney", + city="New South Wales", + timezone="Australia", + language="en", + additional_public_data={ + "first_name": "Steve", + "last_name": "Waugh" + }, + friends=[2, 3] + ) + + achievement = create_achievement() + usersForFriend1 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user1.id) + usersForFriend2 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user2.id) + usersForFriend3 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user3.id) + usersForFriend4 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user4.id) + + print(usersForFriend1) + print(usersForFriend2) + print(usersForFriend3) + print(usersForFriend4) + + self.assertNotIn(4, usersForFriend1) + self.assertNotIn(2, usersForFriend1) + self.assertNotIn(1, usersForFriend2) + self.assertNotIn(3, usersForFriend2) + self.assertNotIn(1, usersForFriend3) + self.assertNotIn(4, usersForFriend3) + self.assertNotIn(2, usersForFriend4) + self.assertNotIn(4, usersForFriend4) + + def test_get_level(self): + return + user = create_user() + + achievement = create_achievement() + + achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) + + update_connection().execute(t_achievements_users.insert().values({ + "user_id": user.id, + "achievement_id": achievement["id"], + "achievement_date": achievement_date, + "level": 1 + })) + + DBSession.flush() + + level_object = achievement.get_level(user.id, achievement["id"], achievement_date) + + # Change achievement date + achievement_monthly = Achievement() + achievement_monthly.name = "invite_users" + achievement_monthly.valid_start = "2016-12-16" + achievement_monthly.valid_end = "2016-12-18" + achievement_monthly.lat = 40.983 + achievement_monthly.lng = 41.562 + achievement_monthly.max_distance = 200000 + achievement_monthly.evaluation = "monthly" + achievement_monthly.relevance = "friends" + achievement_monthly.view_permission = "everyone" + DBSession.add(achievement_monthly) + DBSession.flush() + + achievement_date1 = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement_monthly["evaluation"]) + + level_object1 = achievement.get_level(user.id, achievement["id"], achievement_date1) + + # Test for get_level as integer + level = achievement.get_level_int(user.id, achievement.id, achievement_date) + level1 = achievement.get_level_int(user.id, achievement.id, achievement_date1) + + # Passed cases + self.assertNotEqual(level_object, None) + self.assertEqual(level_object1, []) + self.assertEqual(level, 1) + self.assertEqual(level1, 0) + + # Failed cases + self.assertEqual(level1, 1) + self.assertEqual(level_object, None) + + def test_get_rewards(self): + return + achievement = create_achievement() + + reward = Reward() + reward.name = "badge" + DBSession.add(reward) + DBSession.flush() + + achievement_reward = AchievementReward() + achievement_reward.achievement_id = achievement.id + achievement_reward.reward_id = reward.id + achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level1}.png" + achievement_reward.from_level = 2 + DBSession.add(achievement_reward) + DBSession.flush() + + achievement_reward = AchievementReward() + achievement_reward.achievement_id = achievement.id + achievement_reward.reward_id = reward.id + achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level2}.png" + achievement_reward.from_level = 3 + DBSession.add(achievement_reward) + DBSession.flush() + + rewardlist1 = Achievement.get_rewards(achievement.id, 1) + print(rewardlist1) + + rewardlist2 = Achievement.get_rewards(achievement.id, 2) + print(rewardlist2) + + rewardlist3 = Achievement.get_rewards(achievement.id, 3) + print(rewardlist3) + + self.assertNotEqual(rewardlist2, None) + self.assertNotEqual(rewardlist3, None) + self.assertEqual(rewardlist1, None) + + def test_get_achievement_properties(self): + return + achievement = create_achievement() + + achievementproperty = AchievementProperty() + achievementproperty.name = "xp" + DBSession.add(achievementproperty) + DBSession.flush() + + achievements_achievementproperty = AchievementAchievementProperty() + achievements_achievementproperty.achievement_id = achievement.id + achievements_achievementproperty.property_id = achievementproperty.id + achievements_achievementproperty.value = "5" + achievements_achievementproperty.from_level = 2 + DBSession.add(achievements_achievementproperty) + DBSession.flush() + + result1 = Achievement.get_achievement_properties(achievement.id, 4) + + result2 = Achievement.get_achievement_properties(achievement.id, 1) + + self.assertNotEqual(result1, []) + self.assertNotEqual(result2, []) + + def test_evaluate_achievement(self): + return diff --git a/gengine/app/tests/test_device.py b/gengine/app/tests/test_device.py index 8554d5a..c922594 100644 --- a/gengine/app/tests/test_device.py +++ b/gengine/app/tests/test_device.py @@ -1,16 +1,10 @@ from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_device, update_device, create_achievement, create_variable, create_value, create_goals, create_goal_properties -from gengine.metadata import DBSession -from gengine.app.model import Variable, Value, Achievement, User, t_achievements_users, Reward, AchievementReward, AchievementProperty, AchievementAchievementProperty, Goal -from gengine.base.model import update_connection - - -from sqlalchemy import (and_, select) +from gengine.app.tests.helpers import create_user, create_device, update_device class TestUserDevice(BaseDBTest): def test_create_user_device(self): - return + user = create_user() device = create_device( @@ -28,7 +22,7 @@ def test_create_user_device(self): self.assertTrue(device.app_version == '1.1') def test_update_user_device(self): - return + user = create_user() create_device(user_id=user.id) @@ -46,394 +40,6 @@ def test_update_user_device(self): self.assertTrue(device.push_id == '5126') self.assertTrue(device.app_version == '1.2') - def test_map_variables_to_rules(self): - return - map1 = Variable.map_variables_to_rules() - print(map1) - self.assertNotEqual(map1, None) - - def test_invalidate_caches_for_variable_and_user(self): - return - variable_id = 1, - user_id = 1 - Variable.invalidate_caches_for_variable_and_user(variable_id, user_id) - - goalsandachievements = Variable.map_variables_to_rules().get(variable_id,[]) - - self.assertEqual(goalsandachievements, None) - - def test_increase_value(self): - return - user = create_user() - variable = create_variable() - value = create_value() - - new_value = value.increase_value(variable.name, user, value.value, value.key) - - # Works correctly when removed check for datetime otherwise failed - self.assertNotEqual(value.value, new_value) - - # Includes get_achievement_by_location and get_achievement_by_date - def test_get_achievements_by_location(self): - return - user = create_user() - achievement = create_achievement() - DBSession.flush() - result = achievement.get_achievements_by_user_for_today(user) - print(result) - - # Works when removed goal condition from achievements_by_location and achievements_by_date - self.assertNotEqual(result, None) - - #Need to check by adding goal - - def test_get_relevant_users_by_achievement_friends_and_user(self): - return - #Create First user - user1 = create_user() - - # Create Second user - user2 = create_user( - lat = 85.59, - lng = 65.75, - country = "DE", - region = "Niedersachsen", - city = "Osnabrück", - timezone = "Europe/Berlin", - language = "de", - additional_public_data = { - "first_name" : "Michael", - "last_name" : "Clarke" - } - ) - - # Create Third user - user3 = create_user( - lat = 12.1, - lng = 12.2, - country = "RO", - region = "Transylvania", - city = "Cluj-Napoca", - timezone = "Europe/Bukarest", - language = "en", - additional_public_data = { - "first_name" : "Rudolf", - "last_name" : "Red Nose" - }, - friends=[1, 2] - ) - - # Create Fourth user - user4 = create_user( - lat = 25.56, - lng = 15.89, - country = "AU", - region = "Sydney", - city = "New South Wales", - timezone = "Australia", - language = "en", - additional_public_data = { - "first_name" : "Steve", - "last_name" : "Waugh" - }, - friends=[3] - ) - - achievement = create_achievement() - friendsOfuser1 = achievement.get_relevant_users_by_achievement_and_user(achievement, user1.id) - friendsOfuser3 = achievement.get_relevant_users_by_achievement_and_user(achievement, user3.id) - friendsOfuser4 = achievement.get_relevant_users_by_achievement_and_user(achievement, user4.id) - - self.assertNotIn(1, friendsOfuser1) - self.assertNotIn(2, friendsOfuser1) - self.assertNotIn(4, friendsOfuser3) - self.assertNotIn(2, friendsOfuser3) - self.assertNotIn(3, friendsOfuser4) - self.assertNotIn(2, friendsOfuser4) - - # For the relevance global - achievement1 = Achievement() - achievement1.name = "invite_users" - achievement1.valid_start = "2016-12-16" - achievement1.valid_end = "2016-12-18" - achievement1.lat = 40.983 - achievement1.lng = 41.562 - achievement1.max_distance = 200000 - achievement1.evaluation = "immediately" - achievement1.relevance = "global" - achievement1.view_permission = "everyone" - DBSession.add(achievement1) - - friendsOfuser1 = achievement.get_relevant_users_by_achievement_and_user(achievement1, user3.id) - - self.assertNotIn(1, friendsOfuser1) - - def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): - return - # Create First user - user1 = create_user() - - # Create Second user - user2 = create_user( - lat=85.59, - lng=65.75, - country="DE", - region="Niedersachsen", - city="Osnabrück", - timezone="Europe/Berlin", - language="de", - additional_public_data={ - "first_name": "Michael", - "last_name": "Clarke" - }, - friends = [1] - - ) - - # Create Third user - user3 = create_user( - lat=12.1, - lng=12.2, - country="RO", - region="Transylvania", - city="Cluj-Napoca", - timezone="Europe/Bukarest", - language="en", - additional_public_data={ - "first_name": "Rudolf", - "last_name": "Red Nose" - }, - friends=[1, 2] - ) - - # Create Fourth user - user4 = create_user( - lat=25.56, - lng=15.89, - country="AU", - region="Sydney", - city="New South Wales", - timezone="Australia", - language="en", - additional_public_data={ - "first_name": "Steve", - "last_name": "Waugh" - }, - friends=[2, 3] - ) - - achievement = create_achievement() - usersForFriend1 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user1.id) - usersForFriend2 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user2.id) - usersForFriend3 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user3.id) - usersForFriend4 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user4.id) - - print(usersForFriend1) - print(usersForFriend2) - print(usersForFriend3) - print(usersForFriend4) - - self.assertNotIn(4, usersForFriend1) - self.assertNotIn(2, usersForFriend1) - self.assertNotIn(1, usersForFriend2) - self.assertNotIn(3, usersForFriend2) - self.assertNotIn(1, usersForFriend3) - self.assertNotIn(4, usersForFriend3) - self.assertNotIn(2, usersForFriend4) - self.assertNotIn(4, usersForFriend4) - - def test_get_level(self): - return - user = create_user() - - achievement = create_achievement() - - achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) - - update_connection().execute(t_achievements_users.insert().values({ - "user_id": user.id, - "achievement_id": achievement["id"], - "achievement_date": achievement_date, - "level": 1 - })) - - DBSession.flush() - - level_object = achievement.get_level(user.id, achievement["id"], achievement_date) - - # Change achievement date - achievement_monthly = Achievement() - achievement_monthly.name = "invite_users" - achievement_monthly.valid_start = "2016-12-16" - achievement_monthly.valid_end = "2016-12-18" - achievement_monthly.lat = 40.983 - achievement_monthly.lng = 41.562 - achievement_monthly.max_distance = 200000 - achievement_monthly.evaluation = "monthly" - achievement_monthly.relevance = "friends" - achievement_monthly.view_permission = "everyone" - DBSession.add(achievement_monthly) - DBSession.flush() - - achievement_date1 = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement_monthly["evaluation"]) - - level_object1 = achievement.get_level(user.id, achievement["id"], achievement_date1) - - # Test for get_level as integer - level = achievement.get_level_int(user.id, achievement.id, achievement_date) - level1 = achievement.get_level_int(user.id, achievement.id, achievement_date1) - - # Passed cases - self.assertNotEqual(level_object, None) - self.assertEqual(level_object1, []) - self.assertEqual(level, 1) - self.assertEqual(level1, 0) - - # Failed cases - self.assertEqual(level1, 1) - self.assertEqual(level_object, None) - - def test_get_rewards(self): - return - achievement = create_achievement() - - reward = Reward() - reward.name = "badge" - DBSession.add(reward) - DBSession.flush() - - achievement_reward = AchievementReward() - achievement_reward.achievement_id = achievement.id - achievement_reward.reward_id = reward.id - achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level1}.png" - achievement_reward.from_level = 2 - DBSession.add(achievement_reward) - DBSession.flush() - - achievement_reward = AchievementReward() - achievement_reward.achievement_id = achievement.id - achievement_reward.reward_id = reward.id - achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level2}.png" - achievement_reward.from_level = 3 - DBSession.add(achievement_reward) - DBSession.flush() - - rewardlist1 = Achievement.get_rewards(achievement.id, 1) - print(rewardlist1) - - rewardlist2 = Achievement.get_rewards(achievement.id, 2) - print(rewardlist2) - - rewardlist3 = Achievement.get_rewards(achievement.id, 3) - print(rewardlist3) - - self.assertNotEqual(rewardlist2, None) - self.assertNotEqual(rewardlist3, None) - self.assertEqual(rewardlist1, None) - - def test_get_achievement_properties(self): - return - achievement = create_achievement() - - achievementproperty = AchievementProperty() - achievementproperty.name = "xp" - DBSession.add(achievementproperty) - DBSession.flush() - - achievements_achievementproperty = AchievementAchievementProperty() - achievements_achievementproperty.achievement_id = achievement.id - achievements_achievementproperty.property_id = achievementproperty.id - achievements_achievementproperty.value = "5" - achievements_achievementproperty.from_level = 2 - DBSession.add(achievements_achievementproperty) - DBSession.flush() - - result1 = Achievement.get_achievement_properties(achievement.id, 4) - - result2 = Achievement.get_achievement_properties(achievement.id, 1) - - self.assertNotEqual(result1, []) - self.assertNotEqual(result2, []) - - def test_evaluate_achievement(self): - return - - def test_compute_progress(self): - return - achievement = create_achievement() - goals = create_goals(achievement) - user = create_user() - result1 = Goal.compute_progress(goals[0], achievement, user.id) - print(result1) - result2 = Goal.compute_progress(goals[1], achievement, user.id) - print(result2) - self.assertIsNotNone(result1) - self.assertIsNotNone(result2) - - # If group_by_key attribute for goal is not set, then default value is considered as False and NOT None - # In compute_progress function , group_by_key is compared with None. Is it desired or need to change it to False? - - def test_evaluate_goal(self): - return - achievement = create_achievement() - goal = create_goals(achievement) - user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) - - evaluation_result = Goal.evaluate(goal[0], achievement, achievement_date, user.id, 5, goal_eval_cache_before=False,execute_triggers=True) - print(evaluation_result) - evaluation_result1 = Goal.evaluate(goal[1], achievement, achievement_date, user.id, 2, goal_eval_cache_before=False,execute_triggers=True) - print(evaluation_result1) - - # True cases - self.assertLessEqual(evaluation_result["value"], 25.0) - self.assertEqual(evaluation_result["achieved"], True) - - # False cases - self.assertGreater(evaluation_result1["value"], 0.0) - self.assertEqual(evaluation_result1["achieved"], True) - - # What is "new" in function evaluate()? - - def test_execute_triggers(self): - return - # Function is called inside evaluate_goal function - achievement = create_achievement() - goal = create_goals(achievement) - user = create_user() - print(goal[0]) - level = 5 - previous_goal = Goal.basic_goal_output(goal[0], level - 1).get("goal_goal", 0) - print(previous_goal) - current_goal = Goal.basic_goal_output(goal[0], level).get("goal_goal", 0) - print(current_goal) - - # What is "value" here? - # We are considering parameter value as 0 - value = 0.0 - result = Goal.select_and_execute_triggers(goal[1], user.id, level, current_goal, value, previous_goal) - print(result) # None - # What is expected result? - - def test_get_goal_properties(self): - achievement = create_achievement() - goals = create_goals(achievement) - - create_goal_properties(goals[0].id) - - level = 4 - result = Goal.get_goal_properties(goals[0].id, level) - print(result) - - level1 = 1 - result1 = Goal.get_goal_properties(goals[0].id, level1) - print(result1) - - # True test - self.assertIsNotNone(result) - - # False test - self.assertNotEqual(result1, []) diff --git a/gengine/app/tests/test_goal.py b/gengine/app/tests/test_goal.py index e69de29..c62636c 100644 --- a/gengine/app/tests/test_goal.py +++ b/gengine/app/tests/test_goal.py @@ -0,0 +1,203 @@ +from gengine.app.tests.base import BaseDBTest +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals, create_goal_properties +from gengine.metadata import DBSession +from gengine.app.model import Achievement, User, Goal, GoalEvaluationCache + + +class TestEvaluateGoal(BaseDBTest): + def test_compute_progress(self): + return + achievement = create_achievement() + goals = create_goals(achievement) + user = create_user() + + variable_participate = create_variable("participate", "none") + create_value(user.id, variable_participate.id, 2, "5") + create_value(user.id, variable_participate.id, 3, "7") + create_value(user.id, variable_participate.id, 5, "7") + + variable_invite = create_variable("invite_users", "none") + create_value(user.id, variable_invite.id, 6) + create_value(user.id, variable_invite.id, 7) + + # For goal0, since its group_by_key is false, progress is sum of all the values of Keys 5 and 7 + users_progress_goal0 = Goal.compute_progress(goals[0], achievement, user.id) + goal0_evaluation = {e["user_id"]: e["value"] for e in users_progress_goal0} + print(goal0_evaluation) + + # For goal1, since its group_by_key is True, it'll group by key and add their respective values + users_progress_goal1 = Goal.compute_progress(goals[1], achievement, user.id) + goal1_evaluation = {e["user_id"]: e["value"] for e in users_progress_goal1} + print(goal1_evaluation) + + self.assertLessEqual(goal0_evaluation.get(user.id), 13) + + # Check with group_by_key for goals[1] = False + self.assertLessEqual(goal1_evaluation.get(user.id), 10) + + # Check with group_by_key for goals[1] = True + self.assertLessEqual(goal1_evaluation.get(user.id), 2) + + # If group_by_key attribute for goal is not set, then default value is considered as False and NOT None + # In compute_progress function , group_by_key is compared with None. Is it desired or need to change it to False? + + + def test_evaluate_goal(self): + return + achievement = create_achievement() + goal = create_goals(achievement) + user = create_user() + achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) + + evaluation_result = Goal.evaluate(goal[0], achievement, achievement_date, user.id, 5, goal_eval_cache_before=False, execute_triggers=True) + print(evaluation_result) + evaluation_result1 = Goal.evaluate(goal[1], achievement, achievement_date, user.id, 2, goal_eval_cache_before=False, execute_triggers=True) + print(evaluation_result1) + + # True cases + self.assertLessEqual(evaluation_result["value"], 25.0) + self.assertEqual(evaluation_result["achieved"], True) + + # False cases + self.assertGreater(evaluation_result1["value"], 0.0) + self.assertEqual(evaluation_result1["achieved"], True) + + + def test_execute_triggers(self): + return + # Function is called inside evaluate_goal function + achievement = create_achievement() + goal = create_goals(achievement) + user = create_user() + print(goal[0]) + level = 5 + previous_goal = Goal.basic_goal_output(goal[0], level - 1).get("goal_goal", 0) + print(previous_goal) + current_goal = Goal.basic_goal_output(goal[0], level).get("goal_goal", 0) + print(current_goal) + + # What is "value" here? + # We are considering parameter value as 0 + value = 0.0 + result = Goal.select_and_execute_triggers(goal[1], user.id, level, current_goal, value, previous_goal) + print(result) # None + # What is expected result? + + + def test_get_goal_properties(self): + return + achievement = create_achievement() + goals = create_goals(achievement) + + create_goal_properties(goals[0].id) + + level = 4 + result = Goal.get_goal_properties(goals[0].id, level) + print(result) + + level1 = 1 + result1 = Goal.get_goal_properties(goals[0].id, level1) + print(result1) + + # True test + self.assertIsNotNone(result) + + # False test + # self.assertNotEqual(result1, []) + + + def test_get_leaderboard(self): + return + achievement = create_achievement() + goals = create_goals(achievement) + + # Create multiple users for a goal + user1 = create_user() + user2 = create_user( + lat=85.59, + lng=65.75, + country="DE", + region="Niedersachsen", + city="Osnabrück", + timezone="Europe/Berlin", + language="de", + additional_public_data={ + "first_name": "Michael", + "last_name": "Clarke" + } + ) + + # Create Third user + user3 = create_user( + lat=12.1, + lng=12.2, + country="RO", + region="Transylvania", + city="Cluj-Napoca", + timezone="Europe/Bucharest", + language="en", + additional_public_data={ + "first_name": "Rudolf", + "last_name": "Red Nose" + }, + friends=[1, 2] + ) + + # Create Fourth user + user4 = create_user( + lat=25.56, + lng=15.89, + country="AU", + region="Sydney", + city="New South Wales", + timezone="Australia/Sydney", + language="en", + additional_public_data={ + "first_name": "Steve", + "last_name": "Waugh" + }, + friends=[3] + ) + + achievement_date_for_user1 = Achievement.get_datetime_for_evaluation_type(User.get_user(user1.id)["timezone"], achievement["evaluation"]) + achievement_date_for_user2 = Achievement.get_datetime_for_evaluation_type(User.get_user(user2.id)["timezone"], achievement["evaluation"]) + achievement_date_for_user3 = Achievement.get_datetime_for_evaluation_type(User.get_user(user3.id)["timezone"], achievement["evaluation"]) + achievement_date_for_user4 = Achievement.get_datetime_for_evaluation_type(User.get_user(user4.id)["timezone"], achievement["evaluation"]) + + # Creating entries for goal_evaluation_cache + goal_evaluation_cache = GoalEvaluationCache() + goal_evaluation_cache.goal_id = goals[0].id + goal_evaluation_cache.achievement_date = achievement_date_for_user1 + goal_evaluation_cache.user_id = user1.id + goal_evaluation_cache.achieved = True + goal_evaluation_cache.value = 8.00 + DBSession.add(goal_evaluation_cache) + DBSession.flush() + + goal_evaluation_cache = GoalEvaluationCache() + goal_evaluation_cache.goal_id = goals[0].id + goal_evaluation_cache.achievement_date = achievement_date_for_user2 + goal_evaluation_cache.user_id = user2.id + goal_evaluation_cache.achieved = True + goal_evaluation_cache.value = 6.00 + DBSession.add(goal_evaluation_cache) + DBSession.flush() + + # Test for finding leaderboard in case where goal has been evaluated for all given users + + # First get list of friends (user_ids) of given user + user_ids = Achievement.get_relevant_users_by_achievement_and_user(achievement, user3.id) + + # Get leaderboard + positions = Goal.get_leaderboard(goals[0], achievement_date_for_user3, user_ids) + + self.assertEqual(positions[1]["user"]["additional_public_data"]["last_name"], "Clarke") + self.assertEqual(positions[0]["value"], 8.00) + + # Test for Goal is not evaluated for few user_ids + user_ids = Achievement.get_relevant_users_by_achievement_and_user(achievement, user4.id) + positions = Goal.get_leaderboard(goals[0], achievement_date_for_user4, user_ids) + + print(positions) + self.assertEqual(positions[0]["value"], 8.00) + self.assertEqual(positions[1]["user"]["additional_public_data"]["last_name"], "Clarke") \ No newline at end of file diff --git a/gengine/app/tests/test_value.py b/gengine/app/tests/test_value.py new file mode 100644 index 0000000..f216a23 --- /dev/null +++ b/gengine/app/tests/test_value.py @@ -0,0 +1,35 @@ +from gengine.app.tests.base import BaseDBTest +from gengine.app.tests.helpers import create_user, create_variable, create_value, Variable +from gengine.app.model import Variable, Value + + +class TestValue(BaseDBTest): + def test_increase_value(self): + return + user = create_user() + variable = create_variable("participate", "none") + value_key5 = create_value(user.id, variable.id, 2, "5") + value_key7 = create_value(user.id, variable.id, 3, "7") + + new_value_key5 = Value.increase_value(variable.name, user, 1, value_key5.key) + new_value_key7 = value_key5.increase_value(variable.name, user, 2, value_key7.key) + + # Works correctly when removed check for datetime otherwise failed + self.assertNotEqual(value_key5.value, new_value_key5) + self.assertless(value_key5.value, new_value_key7) + + def test_map_variables_to_rules(self): + return + map1 = Variable.map_variables_to_rules() + print(map1) + self.assertNotEqual(map1, None) + + def test_invalidate_caches_for_variable_and_user(self): + return + variable = create_variable() + user = create_user() + Variable.invalidate_caches_for_variable_and_user(variable.id, user.id) + + goalsandachievements = Variable.map_variables_to_rules().get(variable.id, []) + + self.assertEqual(goalsandachievements, None) diff --git a/gengine/base/model.py b/gengine/base/model.py index 0ef8f4d..146a9ef 100644 --- a/gengine/base/model.py +++ b/gengine/base/model.py @@ -112,6 +112,6 @@ def valid_timezone(timezone): def update_connection(): - session = meta.DBSession() + session = meta.DBSession mark_changed(session) return session From 8bc0c72fc7490abf0aad6127ce9b04d43a38a284 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Mon, 16 Jan 2017 11:02:30 +0100 Subject: [PATCH 111/176] filter values concerning evaluation_type --- gengine/app/model.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index a7c0eb6..c74826e 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1217,14 +1217,17 @@ def generate_statement_cache(): if evaluation_type!="immediately": - if evaluation_type=="day": + if evaluation_type=="daily": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("day","users.timezone"))) - elif evaluation_type=="week": + elif evaluation_type=="weekly": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("week","users.timezone"))) - elif evaluation_type=="month": + elif evaluation_type=="monthly": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("month","users.timezone"))) - elif evaluation_type=="year": + elif evaluation_type=="yearly": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("year","users.timezone"))) + elif evaluation_type == "end": + pass + #Todo implement for end if datetime_col is not None or group_by_key is not None: if datetime_col is not None: From dc5d51f3e7060e8e4618db2840508b24042f057f Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Thu, 19 Jan 2017 18:44:26 +0100 Subject: [PATCH 112/176] Added integration tests for achievement and goal, issues in increase_value function --- gengine/app/model.py | 13 ++-- gengine/app/tests/helpers.py | 14 ++-- gengine/app/tests/test_achievement1.py | 93 +++++++++++++++++++++++++- gengine/app/tests/test_auth.py | 38 ++++++++--- gengine/app/tests/test_value.py | 34 +++------- 5 files changed, 140 insertions(+), 52 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 7ff8eb3..41b46db 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -689,7 +689,8 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): variable = Variable.get_variable_by_name(variable_name) dt = Variable.get_datetime_for_tz_and_group(tz,variable["group"],at_datetime=at_datetime) - + print("datetime ",t_values.c.datetime) + print("dt ",dt) condition = and_(t_values.c.datetime==dt, t_values.c.variable_id==variable["id"], t_values.c.user_id==user_id, @@ -912,6 +913,7 @@ def generate(): if not goal_eval: Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level,None) goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) + print("Goal eval ",goal_eval) if achievement["relevance"]=="friends" or achievement["relevance"]=="city" or achievement["relevance"]=="global": goal_eval["leaderboard"] = Goal.get_leaderboard(goal, achievement_date, user_ids) @@ -1230,8 +1232,6 @@ def generate_statement_cache(): elif evaluation_type=="year": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("year","users.timezone"))) - print("datetime_col ", datetime_col) - print("group_by_key ", group_by_key) if datetime_col is not None or group_by_key is not False: if datetime_col is not None: q = q.group_by(datetime_col) @@ -1257,19 +1257,16 @@ def generate_statement_cache(): #q = cache_goal_statements.get_or_create(str(goal["id"]),generate_statement_cache) # TODO: Cache the statement / Make it serializable for caching in redis q = generate_statement_cache() - print("progress") - print(DBSession.execute(q, {'user_id' : user_id})) return DBSession.execute(q, {'user_id' : user_id}) @classmethod def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval_cache_before=False, execute_triggers=True): """evaluate the goal for the user_ids and the level""" - operator = goal["operator"] users_progress = Goal.compute_progress(goal, achievement, user_id) goal_evaluation = {e["user_id"] : e["value"] for e in users_progress} - + print("user_progress ", goal_evaluation) goal_achieved = False if goal_eval_cache_before is False: @@ -1286,8 +1283,10 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval print("params", params) goal_goal = evaluate_value_expression(goal["goal"], params) print("goal_goal",goal_goal) + print("user_value ", new) if goal_goal is not None and operator=="geq" and new>=goal_goal: goal_achieved = True + print("goal_achieved is ",goal_achieved) new = min(new,goal_goal) elif goal_goal is not None and operator=="leq" and new<=goal_goal: diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index e20db0f..8817e51 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -1,11 +1,11 @@ import names import random -from gengine.app.model import User, Language, Achievement,Goal, Variable, Value, t_goals, GoalProperty, GoalGoalProperty, TranslationVariable, t_goals_goalproperties +from gengine.app.model import User, Language, Achievement,Goal, Variable, Value, t_goals, GoalProperty, GoalGoalProperty, TranslationVariable, t_goals_goalproperties, t_users from gengine.metadata import DBSession from gengine.app.model import UserDevice, t_user_device -from sqlalchemy import (and_) +from sqlalchemy import and_, select default_gen_data = { "timezone" : "Europe/Berlin", @@ -32,10 +32,10 @@ } default_device_data = { - "device_id" : "1234", "device_os" : "iOS 5", "app_version" : "1.1", - "push_id" : "5678" + "push_id" : "5678", + "device_id" : "1234" } class Undefined(): @@ -142,8 +142,9 @@ def delete_user( ): User.delete_user(user_id) - - return User.get_user(user_id) + users = DBSession.execute(select([t_users.c.id,])).fetchall() + print(users) + return users def get_or_create_language(name): @@ -225,6 +226,7 @@ def create_achievement(): achievement.max_distance = 200000 achievement.evaluation = "immediately" achievement.relevance = "friends" + achievement.maxlevel = 3 achievement.view_permission = "everyone" DBSession.add(achievement) diff --git a/gengine/app/tests/test_achievement1.py b/gengine/app/tests/test_achievement1.py index f8954d4..035855f 100644 --- a/gengine/app/tests/test_achievement1.py +++ b/gengine/app/tests/test_achievement1.py @@ -1,7 +1,7 @@ from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_achievement +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals from gengine.metadata import DBSession -from gengine.app.model import Achievement, User +from gengine.app.model import Achievement, User, AchievementUser, Goal, Value class TestAchievement(BaseDBTest): @@ -286,5 +286,92 @@ def test_get_achievement_properties(self): self.assertNotEqual(result1, []) self.assertNotEqual(result2, []) - def test_evaluate_achievement(self): + def test_evaluate_achievement_for_participate(self): return + # Achievement with relevance own and maxlevel 3 + achievement = create_achievement() + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + + # get level + current_level = 2 + achievement_user = AchievementUser() + achievement_user.user_id = user.id + achievement_user.achievement_id = achievement.id + achievement_user.achievement_date = achievement_date + achievement_user.level = current_level + DBSession.add(achievement_user) + DBSession.flush() + + variable = create_variable("participate", "none") + + goal = Goal() + goal.condition = """{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""" + goal.goal = "1*level" + goal.group_by_key = True + goal.operator = "geq" + goal.achievement_id = achievement.id + DBSession.add(goal) + DBSession.flush() + + Achievement.evaluate(user, achievement.id, achievement_date).get("level") + Value.increase_value(variable_name="participate", user=user, value=1, key="5") + Achievement.evaluate(user, achievement.id, achievement_date).get("level") + Value.increase_value(variable_name="participate", user=user, value=1, key="5") + Achievement.evaluate(user, achievement.id, achievement_date).get("level") + Value.increase_value(variable_name="participate", user=user, value=1, key="5") + result = Achievement.evaluate(user, achievement.id, achievement_date) + print(result) + + self.assertEqual(result["level"], achievement.maxlevel) + + def test_evaluate_achievement_for_invite_users(self): + return + # Achievement with relevance friends and maxlevel 3 + achievement = create_achievement() + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + + # get level + current_level = 1 + achievement_user = AchievementUser() + achievement_user.user_id = user.id + achievement_user.achievement_id = achievement.id + achievement_user.achievement_date = achievement_date + achievement_user.level = current_level + DBSession.add(achievement_user) + DBSession.flush() + + variable = create_variable("invite_users", "none") + firstvalue = Value.increase_value(variable_name="invite_users", user=user, value=1, key="5") + print("firstvalue ", firstvalue) + DBSession.flush() + + goal = Goal() + goal.condition = """{"term": {"type": "literal", "variable": "invite_users"}}""" + goal.goal = "3*level" + goal.operator = "geq" + goal.group_by_key = False + goal.achievement_id = achievement.id + DBSession.add(goal) + DBSession.flush() + + level = Achievement.evaluate(user, achievement.id, achievement_date).get("level") + print("level ",level) + new2 = Value.increase_value(variable_name="invite_users", user=user, value=8, key="5") + print("new2 ",new2) + level2 = Achievement.evaluate(user, achievement.id, achievement_date) + new = Value.increase_value(variable.name, user, 7, key="5") + print("level2 ", level2) + x = Goal.evaluate(goal,achievement,achievement_date, user.id, Achievement.evaluate(user, achievement.id, achievement_date).get("level")) + print("goal eval",x) + level3 = Achievement.evaluate(user, achievement.id, achievement_date) + print("level3 ",level3) + + + + diff --git a/gengine/app/tests/test_auth.py b/gengine/app/tests/test_auth.py index fd7ec37..5a38648 100644 --- a/gengine/app/tests/test_auth.py +++ b/gengine/app/tests/test_auth.py @@ -6,7 +6,7 @@ class TestUserCreation(BaseDBTest): def test_user_creation(self): - return + lang = get_or_create_language("en") user = create_user( @@ -34,7 +34,7 @@ def test_user_creation(self): self.assertTrue(user.additional_public_data["last_name"] == "Red Nose") def test_user_updation(self): - return + lang = get_or_create_language("en") user = create_user() user = update_user( @@ -63,18 +63,34 @@ def test_user_updation(self): self.assertTrue(user.additional_public_data["last_name"] == "Red Nose") def test_user_deletion(self): - return - lang = get_or_create_language("en") - user = create_user() - user = delete_user( - user_id = user.id + # Create First user + user1 = create_user() + + # Create Second user + user2 = create_user( + lat=85.59, + lng=65.75, + country="DE", + region="Niedersachsen", + city="Osnabrück", + timezone="Europe/Berlin", + language="de", + additional_public_data={ + "first_name": "Michael", + "last_name": "Clarke" + }, + friends=[1] ) - self.assertE(user, None) + remainingusers = delete_user( + user_id = user1.id + ) + + self.assertNotIn(user1.id, remainingusers) def test_verify_password(self): - return + auth_user = AuthUser() auth_user.password = "test12345" auth_user.active = True @@ -82,12 +98,12 @@ def test_verify_password(self): DBSession.add(auth_user) iscorrect = auth_user.verify_password("test12345") - print(isCorrect) + print(iscorrect) self.assertEqual(iscorrect, True) def test_create_token(self): - return + user = create_user() auth_user = AuthUser() auth_user.user_id = user.id diff --git a/gengine/app/tests/test_value.py b/gengine/app/tests/test_value.py index f216a23..e3207fe 100644 --- a/gengine/app/tests/test_value.py +++ b/gengine/app/tests/test_value.py @@ -1,35 +1,19 @@ from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_variable, create_value, Variable -from gengine.app.model import Variable, Value +from gengine.app.tests.helpers import create_user, create_variable,create_value +from gengine.app.model import Value class TestValue(BaseDBTest): def test_increase_value(self): - return user = create_user() variable = create_variable("participate", "none") - value_key5 = create_value(user.id, variable.id, 2, "5") - value_key7 = create_value(user.id, variable.id, 3, "7") + key = "5" + value = create_value(user.id, variable.id, 1, key) - new_value_key5 = Value.increase_value(variable.name, user, 1, value_key5.key) - new_value_key7 = value_key5.increase_value(variable.name, user, 2, value_key7.key) + value1 = Value.increase_value(variable.name, user, 3, key) + value2 = Value.increase_value(variable.name, user, 2, key) - # Works correctly when removed check for datetime otherwise failed - self.assertNotEqual(value_key5.value, new_value_key5) - self.assertless(value_key5.value, new_value_key7) + self.assertGreater(value2, value1) - def test_map_variables_to_rules(self): - return - map1 = Variable.map_variables_to_rules() - print(map1) - self.assertNotEqual(map1, None) - - def test_invalidate_caches_for_variable_and_user(self): - return - variable = create_variable() - user = create_user() - Variable.invalidate_caches_for_variable_and_user(variable.id, user.id) - - goalsandachievements = Variable.map_variables_to_rules().get(variable.id, []) - - self.assertEqual(goalsandachievements, None) + # Problems in increase_value function. May be because datetime check. + # Increase value is being called only in evaluate_achievement function and not in evaluate_goal \ No newline at end of file From 57f4c607d3049bcc5a032a75a9f940008d7609d5 Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Thu, 26 Jan 2017 18:48:31 +0100 Subject: [PATCH 113/176] Adding integration tests,changing helper class, code refactoring --- gengine/app/model.py | 30 ++-- gengine/app/tests/helpers.py | 165 +++++++++++++++---- gengine/app/tests/test_achievement1.py | 215 +++++++++++++++++++------ gengine/app/tests/test_value.py | 1 + 4 files changed, 317 insertions(+), 94 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 41b46db..c2e585e 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -689,8 +689,6 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): variable = Variable.get_variable_by_name(variable_name) dt = Variable.get_datetime_for_tz_and_group(tz,variable["group"],at_datetime=at_datetime) - print("datetime ",t_values.c.datetime) - print("dt ",dt) condition = and_(t_values.c.datetime==dt, t_values.c.variable_id==variable["id"], t_values.c.user_id==user_id, @@ -744,7 +742,6 @@ def generate_achievements_by_user_for_today(): today = datetime.date.today() by_loc = {x["id"] : x["distance"] for x in cls.get_achievements_by_location(coords(user))} by_date = cls.get_achievements_by_date(today) - def update(arr,distance): arr["distance"]=distance return arr @@ -764,13 +761,12 @@ def get_achievements_by_location(cls,latlng): """return achievements which are valid in that location.""" #TODO: invalidate automatically when achievement in user's range is modified distance = calc_distance(latlng, (t_achievements.c.lat, t_achievements.c.lng)).label("distance") - q = select([t_achievements.c.id, distance])\ .where(or_(and_(t_achievements.c.lat==None,t_achievements.c.lng==None), distance < t_achievements.c.max_distance)) - return [dict(x.items()) for x in DBSession.execute(q).fetchall() if len(Goal.get_goals(x['id']))>0] + return [dict(x.items()) for x in DBSession.execute(q).fetchall() if len(Goal.get_goals(x['id'])) > 0] @classmethod @cache_general.cache_on_arguments() @@ -781,6 +777,7 @@ def get_achievements_by_date(cls,date): or_(t_achievements.c.valid_end==None, t_achievements.c.valid_end>=date) )) + return [dict(x.items()) for x in DBSession.execute(q).fetchall() if len(Goal.get_goals(x['id']))>0] #TODO:CACHE @@ -893,8 +890,10 @@ def evaluate(cls, user, achievement_id, achievement_date): return the basic_output for the achievement plus information about the new achieved levels """ - + print("In evaluate achie") def generate(): + print("In evaluate achie generate") + achievement = Achievement.get_achievement(achievement_id) user_id = user["id"] @@ -906,9 +905,8 @@ def generate(): goal_evals={} all_goals_achieved = True goals = Goal.get_goals(achievement["id"]) - for goal in goals: - + print("For goal in goals") goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) if not goal_eval: Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level,None) @@ -1047,7 +1045,20 @@ def get_rewards(cls,achievement_id,level): @cache_general.cache_on_arguments() def get_achievement_properties(cls,achievement_id,level): """return all properties which are associated to the achievement level.""" - + print("user_wants_level property function ",level) + result = DBSession.execute(select([t_achievementproperties.c.id.label("property_id"), + t_achievementproperties.c.name, + t_achievementproperties.c.is_variable, + t_achievements_achievementproperties.c.from_level, + t_achievements_achievementproperties.c.value, + t_achievements_achievementproperties.c.value_translation_id], + from_obj=t_achievementproperties.join(t_achievements_achievementproperties))\ + .where(and_(or_(t_achievements_achievementproperties.c.from_level<=level, + t_achievements_achievementproperties.c.from_level==None), + t_achievements_achievementproperties.c.achievement_id==achievement_id))\ + .order_by(t_achievements_achievementproperties.c.from_level))\ + .fetchall() + print("Property result in model ",result) return DBSession.execute(select([t_achievementproperties.c.id.label("property_id"), t_achievementproperties.c.name, t_achievementproperties.c.is_variable, @@ -1377,7 +1388,6 @@ def select_and_execute_triggers(cls, goal, user_id, level, current_goal, value, def get_goal_eval_cache(cls,goal_id,achievement_date,user_id): """lookup and return cache entry, else return None""" v = cache_goal_evaluation.get("%s_%s_%s" % (goal_id,achievement_date,user_id)) - print(v) if v: return v else: diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index 8817e51..fcb73a5 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -1,5 +1,6 @@ import names import random +import datetime from gengine.app.model import User, Language, Achievement,Goal, Variable, Value, t_goals, GoalProperty, GoalGoalProperty, TranslationVariable, t_goals_goalproperties, t_users from gengine.metadata import DBSession @@ -143,7 +144,6 @@ def delete_user( User.delete_user(user_id) users = DBSession.execute(select([t_users.c.id,])).fetchall() - print(users) return users @@ -216,51 +216,148 @@ def update_device( return device -def create_achievement(): +def create_achievement( + achievement_name = undefined, + achievement_valid_start = undefined, + achievement_valid_end = undefined, + achievement_lat = undefined, + achievement_lng = undefined, + achievement_max_distance = undefined, + achievement_evaluation = undefined, + achievement_relevance = undefined, + achievement_maxlevel = undefined, + achievement_view_permission = undefined, + ): achievement = Achievement() - achievement.name = "invite_users" - achievement.valid_start = "2016-12-16" - achievement.valid_end = "2016-12-18" - achievement.lat = 40.983 - achievement.lng = 41.562 - achievement.max_distance = 200000 - achievement.evaluation = "immediately" - achievement.relevance = "friends" - achievement.maxlevel = 3 - achievement.view_permission = "everyone" - DBSession.add(achievement) - DBSession.flush() + if achievement_name is undefined: + achievement.name = "invite_users_achievement" + else: + achievement.name = achievement_name + + if achievement_valid_start is undefined: + achievement.valid_start = "2016-12-16" + else: + achievement.valid_start = achievement_valid_start + + if achievement_valid_end is undefined: + achievement.valid_end = datetime.datetime.utcnow() + else: + achievement.valid_end = achievement_valid_end + + if achievement_lat is undefined: + achievement.lat = 40.983 + else: + achievement.lat = achievement_lat + + if achievement_lng is undefined: + achievement.lng = 41.562 + else: + achievement.lng = achievement_lng + + if achievement_max_distance is undefined: + achievement.max_distance = 20000 + else: + achievement.max_distance = achievement_max_distance + + if achievement_evaluation is undefined: + achievement.evaluation = "immediately" + else: + achievement.evaluation = achievement_evaluation + + if achievement_relevance is undefined: + achievement.relevance = "friends" + else: + achievement.relevance = achievement_relevance + + if achievement_maxlevel is undefined: + achievement.maxlevel = 3 + else: + achievement.maxlevel = achievement_maxlevel + + if achievement_view_permission is undefined: + achievement.view_permission = "everyone" + else: + achievement.view_permission = achievement_view_permission - achievement = achievement.get_achievement(achievement.id) + DBSession.add(achievement) DBSession.flush() return achievement -def create_goals(achievement): +def create_goals( + achievement = undefined, + goal_condition = undefined, + goal_goal = undefined, + goal_operator = undefined, + goal_group_by_key = undefined, + goal_name = undefined + ): goal = Goal() - goal.condition = """{"term": {"type": "literal", "variable": "invite_users"}}""" - goal.goal = "5*level" - goal.operator = "leq" - goal.achievement_id = achievement.id - DBSession.add(goal) - DBSession.flush() + if achievement["name"] is "invite_users_achievement": + + if goal_condition is undefined: + goal.condition = """{"term": {"type": "literal", "variable": "invite_users"}}""" + else: + goal.condition = goal_condition + + if goal_goal is undefined: + goal.goal = "5*level" + else: + goal.goal = goal_goal + + if goal_operator is undefined: + goal.operator = "geq" + else: + goal.operator = goal_operator + + if goal_group_by_key is undefined: + goal.group_by_key = False + else: + goal.group_by_key = goal_group_by_key + + if goal_name is undefined: + goal.name = "goal_invite_users" + else: + goal.name = goal_name + + goal.achievement_id = achievement.id + DBSession.add(goal) + DBSession.flush() - goal1 = Goal() - goal1.condition = """{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""" - goal1.goal = "3*level" - goal1.group_by_key = False - goal1.operator = "geq" - goal1.achievement_id = achievement.id - DBSession.add(goal1) - DBSession.flush() + if achievement["name"] is "participate_achievement": - goals = DBSession.execute(t_goals.select(t_goals.c.achievement_id == achievement.id)).fetchall() - DBSession.flush() - print(goals) + if goal_condition is undefined: + goal.condition = """{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""" + else: + goal.condition = goal_condition + + if goal_goal is undefined: + goal.goal = "5*level" + else: + goal.goal = goal_goal + + if goal_operator is undefined: + goal.operator = "geq" + else: + goal.operator = goal_operator + + if goal_group_by_key is undefined: + goal.group_by_key = True + else: + goal.group_by_key = goal_group_by_key + + if goal_name is undefined: + goal.name = "goal_participate" + else: + goal.name = goal_name + + goal.achievement_id = achievement.id + DBSession.add(goal) + DBSession.flush() - return goals + return goal def create_goal_properties(goal_id): diff --git a/gengine/app/tests/test_achievement1.py b/gengine/app/tests/test_achievement1.py index 035855f..37b8f45 100644 --- a/gengine/app/tests/test_achievement1.py +++ b/gengine/app/tests/test_achievement1.py @@ -1,23 +1,25 @@ +import datetime from gengine.app.tests.base import BaseDBTest from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals from gengine.metadata import DBSession -from gengine.app.model import Achievement, User, AchievementUser, Goal, Value +from gengine.app.model import Achievement, User, AchievementUser, Goal, Value, AchievementReward, Reward, AchievementProperty, AchievementAchievementProperty class TestAchievement(BaseDBTest): # Includes get_achievement_by_location and get_achievement_by_date - def test_get_achievements_by_location(self): + def test_get_achievements_by_location_and_date(self): return user = create_user() - achievement = create_achievement() + achievement1 = create_achievement(achievement_name="invite_users_achievement") + achievement2 = create_achievement(achievement_name="participate_achievement") DBSession.flush() - result = achievement.get_achievements_by_user_for_today(user) - print(result) + create_goals(achievement1) + achievement_today = Achievement.get_achievements_by_user_for_today(user) + print(achievement_today) - # Works when removed goal condition from achievements_by_location and achievements_by_date - self.assertNotEqual(result, None) + self.assertEqual(achievement_today[0]["name"], "invite_users_achievement") + self.assertEqual(len(achievement_today), 2) - #Need to check by adding goal def test_get_relevant_users_by_achievement_friends_and_user(self): return @@ -84,16 +86,7 @@ def test_get_relevant_users_by_achievement_friends_and_user(self): self.assertNotIn(2, friendsOfuser4) # For the relevance global - achievement1 = Achievement() - achievement1.name = "invite_users" - achievement1.valid_start = "2016-12-16" - achievement1.valid_end = "2016-12-18" - achievement1.lat = 40.983 - achievement1.lng = 41.562 - achievement1.max_distance = 200000 - achievement1.evaluation = "immediately" - achievement1.relevance = "global" - achievement1.view_permission = "everyone" + achievement1 = create_achievement(achievement_relevance = "global") DBSession.add(achievement1) friendsOfuser1 = achievement.get_relevant_users_by_achievement_and_user(achievement1, user3.id) @@ -193,16 +186,7 @@ def test_get_level(self): level_object = achievement.get_level(user.id, achievement["id"], achievement_date) # Change achievement date - achievement_monthly = Achievement() - achievement_monthly.name = "invite_users" - achievement_monthly.valid_start = "2016-12-16" - achievement_monthly.valid_end = "2016-12-18" - achievement_monthly.lat = 40.983 - achievement_monthly.lng = 41.562 - achievement_monthly.max_distance = 200000 - achievement_monthly.evaluation = "monthly" - achievement_monthly.relevance = "friends" - achievement_monthly.view_permission = "everyone" + achievement_monthly = create_achievement(achievement_evaluation = "monthly") DBSession.add(achievement_monthly) DBSession.flush() @@ -258,8 +242,11 @@ def test_get_rewards(self): rewardlist3 = Achievement.get_rewards(achievement.id, 3) print(rewardlist3) + # passed test cases self.assertNotEqual(rewardlist2, None) self.assertNotEqual(rewardlist3, None) + + # failed test cases self.assertEqual(rewardlist1, None) def test_get_achievement_properties(self): @@ -280,23 +267,23 @@ def test_get_achievement_properties(self): DBSession.flush() result1 = Achievement.get_achievement_properties(achievement.id, 4) + print(result1) result2 = Achievement.get_achievement_properties(achievement.id, 1) + print(result2) self.assertNotEqual(result1, []) self.assertNotEqual(result2, []) def test_evaluate_achievement_for_participate(self): return - # Achievement with relevance own and maxlevel 3 - achievement = create_achievement() + achievement = create_achievement(achievement_name="participate_achievement", achievement_relevance="own", achievement_maxlevel=4) user = create_user() achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) - # get level - current_level = 2 + current_level = 1 achievement_user = AchievementUser() achievement_user.user_id = user.id achievement_user.achievement_id = achievement.id @@ -306,31 +293,31 @@ def test_evaluate_achievement_for_participate(self): DBSession.flush() variable = create_variable("participate", "none") + Value.increase_value(variable_name=variable.name, user=user, value=1, key="5") - goal = Goal() - goal.condition = """{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""" - goal.goal = "1*level" - goal.group_by_key = True - goal.operator = "geq" - goal.achievement_id = achievement.id - DBSession.add(goal) - DBSession.flush() + goal = create_goals(achievement, + goal_condition="""{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""", + goal_group_by_key=True, + goal_operator="geq", + goal_goal="1*level") - Achievement.evaluate(user, achievement.id, achievement_date).get("level") - Value.increase_value(variable_name="participate", user=user, value=1, key="5") - Achievement.evaluate(user, achievement.id, achievement_date).get("level") - Value.increase_value(variable_name="participate", user=user, value=1, key="5") - Achievement.evaluate(user, achievement.id, achievement_date).get("level") - Value.increase_value(variable_name="participate", user=user, value=1, key="5") - result = Achievement.evaluate(user, achievement.id, achievement_date) - print(result) + level = Achievement.evaluate(user, achievement.id, achievement_date).get("level") + print("level ",level) + result1 = Value.increase_value(variable_name="participate", user=user, value=5, key="5") + print("value result ",result1) + level2 = Achievement.evaluate(user, achievement.id, achievement_date) + print("level2 ",level2) + result2 = Value.increase_value(variable_name="participate", user=user, value=1, key="5") + # result3 = Value.increase_value(variable_name="participate", user=user, value=1, key="5") + # result = Achievement.evaluate(user, achievement.id, achievement_date) - self.assertEqual(result["level"], achievement.maxlevel) + #self.assertEqual(result["level"], achievement.maxlevel) def test_evaluate_achievement_for_invite_users(self): return + # increase_value function: may need to remove time-constraint, because difference of millisecond fail to increase the value next time # Achievement with relevance friends and maxlevel 3 - achievement = create_achievement() + achievement = create_achievement(achievement_relevance="friends", achievement_maxlevel=3) user = create_user() @@ -372,6 +359,134 @@ def test_evaluate_achievement_for_invite_users(self): level3 = Achievement.evaluate(user, achievement.id, achievement_date) print("level3 ",level3) + def test_get_reward_and_properties_for_achievement(self): + return + user = create_user() + + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3) + + # Check for property + achievementproperty = AchievementProperty() + achievementproperty.name = "xp" + DBSession.add(achievementproperty) + DBSession.flush() + + achievements_achievementproperty = AchievementAchievementProperty() + achievements_achievementproperty.achievement_id = achievement.id + achievements_achievementproperty.property_id = achievementproperty.id + achievements_achievementproperty.value = "5" + achievements_achievementproperty.from_level = None + DBSession.add(achievements_achievementproperty) + DBSession.flush() + + reward = Reward() + reward.name = "badge" + DBSession.add(reward) + DBSession.flush() + + achievement_reward = AchievementReward() + achievement_reward.achievement_id = achievement.id + achievement_reward.reward_id = reward.id + achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level1}.png" + achievement_reward.from_level = 2 + DBSession.add(achievement_reward) + DBSession.flush() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + + current_level = 1 + achievement_user = AchievementUser() + achievement_user.user_id = user.id + achievement_user.achievement_id = achievement.id + achievement_user.achievement_date = achievement_date + achievement_user.level = current_level + DBSession.add(achievement_user) + DBSession.flush() + + variable = create_variable("invite_users", "none") + firstvalue = Value.increase_value(variable_name="invite_users", user=user, value=2, key="5") + DBSession.flush() + + goal = create_goals(achievement = achievement, + goal_condition="""{"term": {"type": "literal", "variable": "invite_users"}}""", + goal_group_by_key=True, + goal_operator="geq", + goal_goal="1*level") + + result = Achievement.evaluate(user, achievement.id, achievement_date) + print(result) + + self.assertNotEqual(len(result["new_levels"]["2"]["rewards"]), 0) + + Value.increase_value(variable_name="invite_users", user=user, value=2, key="5") + DBSession.flush() + + # Successive calls to Achievement.evaluate function are not working. + # result = Achievement.evaluate(user, achievement.id, achievement_date) + # print(result) + + # result1 = Achievement.get_achievement_properties(achievement.id, 4) + # print("in test property",result1) + + self.assertNotEqual(len(result["new_levels"]["2"]["properties"]), 0) + + def test_multiple_goals_of_same_achievement(self): + + user = create_user() + + achievement = create_achievement(achievement_name="participate_achievement", achievement_maxlevel=3) + + reward = Reward() + reward.name = "badge" + DBSession.add(reward) + DBSession.flush() + + achievement_reward = AchievementReward() + achievement_reward.achievement_id = achievement.id + achievement_reward.reward_id = reward.id + achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level1}.png" + achievement_reward.from_level = 2 + DBSession.add(achievement_reward) + DBSession.flush() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + goal1 = create_goals(achievement=achievement, + goal_condition="""{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate_seminar"}}""", + goal_group_by_key=False, + goal_operator="geq", + goal_goal="2*level", + goal_name = "goal_participate_seminar") + goal2 = create_goals(achievement=achievement, + goal_condition="""{"term": {"type": "literal", "variable": "participate_talk"}}""", + goal_group_by_key=False, + goal_operator="geq", + goal_goal="1*level", + goal_name="goal_participate_talk") + current_level = 1 + achievement_user = AchievementUser() + achievement_user.user_id = user.id + achievement_user.achievement_id = achievement.id + achievement_user.achievement_date = achievement_date + achievement_user.level = current_level + DBSession.add(achievement_user) + DBSession.flush() + + variable1 = create_variable("participate_seminar",variable_group=None) + variable2 = create_variable("participate_talk", variable_group=None) + Value.increase_value(variable1.name, user, "2", "5") + Value.increase_value(variable1.name, user, "2", "7") + Value.increase_value(variable2.name, user, "2", key=None) + + result = Achievement.evaluate(user, achievement.id, achievement_date) + print(result) + Value.increase_value(variable1.name, user, "2", "7") + result1 = Achievement.evaluate(user, achievement.id, achievement_date) + print(result1) + Value.increase_value(variable2.name, user, "2", key=None) + result2 = Achievement.evaluate(user, achievement.id, achievement_date) + print(result2) + + self.assertNotEqual(len(result["new_levels"]["2"]["rewards"]), 0) \ No newline at end of file diff --git a/gengine/app/tests/test_value.py b/gengine/app/tests/test_value.py index e3207fe..7d5f553 100644 --- a/gengine/app/tests/test_value.py +++ b/gengine/app/tests/test_value.py @@ -5,6 +5,7 @@ class TestValue(BaseDBTest): def test_increase_value(self): + return user = create_user() variable = create_variable("participate", "none") key = "5" From 7c27571d937360e1783a3e24c104074f1caf6798 Mon Sep 17 00:00:00 2001 From: Marcel Date: Mon, 30 Jan 2017 11:06:45 +0100 Subject: [PATCH 114/176] dont log pushtext --- gengine/app/push.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gengine/app/push.py b/gengine/app/push.py index 342eff6..e8879b4 100644 --- a/gengine/app/push.py +++ b/gengine/app/push.py @@ -181,7 +181,7 @@ def send_push_message( else: payload = Payload(alert=ios_text, custom=data, badge=message_count, sound="default") - log.debug("Sending Push message to User (ID: %s): %s", user_id, ios_text) + log.debug("Sending Push message to User (ID: %s)", user_id) if device.push_id.startswith("prod_"): get_prod_apns().gateway_server.send_notification(device.push_id[5:], payload, identifier=identifier) @@ -190,7 +190,7 @@ def send_push_message( if "android" in device.device_os.lower(): - log.debug("Sending Push message to User (ID: %s): %s", user_id, android_text) + log.debug("Sending Push message to User (ID: %s)", user_id) push_id = device.push_id.lstrip("dev_").lstrip("prod_") response = get_gcm().json_request(registration_ids=[push_id, ], From ae84fe961497b9cbd4c40e49350753452b72888c Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Tue, 31 Jan 2017 23:10:48 +0100 Subject: [PATCH 115/176] init caches before generating revision --- gengine/maintenance/scripts/generate_revision.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gengine/maintenance/scripts/generate_revision.py b/gengine/maintenance/scripts/generate_revision.py index 7e27a41..ba5dca1 100644 --- a/gengine/maintenance/scripts/generate_revision.py +++ b/gengine/maintenance/scripts/generate_revision.py @@ -54,6 +54,9 @@ def revision(settings, message, options): init_declarative_base() init_db(engine) + from gengine.app.cache import init_caches + init_caches() + from gengine.metadata import ( Base, ) From 2979ffaf4eb9a23806fe003cadae983d5c557888 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Tue, 31 Jan 2017 23:11:57 +0100 Subject: [PATCH 116/176] Fix the PK-constraints concerning recurring achievements --- .../65c7a32b7322_achievement_date_unique.py | 43 +++++++++++++++++ gengine/app/model.py | 46 +++++++++++++++++-- 2 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 gengine/app/alembic/versions/65c7a32b7322_achievement_date_unique.py diff --git a/gengine/app/alembic/versions/65c7a32b7322_achievement_date_unique.py b/gengine/app/alembic/versions/65c7a32b7322_achievement_date_unique.py new file mode 100644 index 0000000..c3e665d --- /dev/null +++ b/gengine/app/alembic/versions/65c7a32b7322_achievement_date_unique.py @@ -0,0 +1,43 @@ +"""achievement_date_unique + +Revision ID: 65c7a32b7322 +Revises: d4a70083f72e +Create Date: 2017-01-31 23:01:11.744725 + +""" + +# revision identifiers, used by Alembic. +revision = '65c7a32b7322' +down_revision = 'd4a70083f72e' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.execute("ALTER TABLE achievements_users DROP CONSTRAINT pk_achievements_users;") + op.execute("ALTER TABLE achievements_users ADD COLUMN id SERIAL;") + op.execute("ALTER TABLE achievements_users ADD CONSTRAINT pk_achievements_users PRIMARY KEY(id);") + + op.execute("ALTER TABLE goal_evaluation_cache DROP CONSTRAINT pk_goal_evaluation_cache;") + op.execute("ALTER TABLE goal_evaluation_cache ADD COLUMN id SERIAL;") + op.execute("ALTER TABLE goal_evaluation_cache ADD CONSTRAINT pk_goal_evaluation_cache PRIMARY KEY(id);") + + op.create_index('idx_achievements_users_date_not_null_unique', 'achievements_users', ['user_id', 'achievement_id', 'achievement_date', 'level'], unique=True, postgresql_where=sa.text('achievement_date IS NOT NULL')) + op.create_index('idx_achievements_users_date_null_unique', 'achievements_users', ['user_id', 'achievement_id', 'level'], unique=True, postgresql_where=sa.text('achievement_date IS NULL')) + op.create_index(op.f('ix_achievements_users_achievement_id'), 'achievements_users', ['achievement_id'], unique=False) + op.create_index(op.f('ix_achievements_users_level'), 'achievements_users', ['level'], unique=False) + + op.create_index('idx_goal_evaluation_cache_date_not_null_unique', 'goal_evaluation_cache', ['user_id', 'goal_id', 'achievement_date'], unique=True, postgresql_where=sa.text('achievement_date IS NOT NULL')) + op.create_index('idx_goal_evaluation_cache_date_null_unique', 'goal_evaluation_cache', ['user_id', 'goal_id'], unique=True, postgresql_where=sa.text('achievement_date IS NULL')) + op.create_index(op.f('ix_goal_evaluation_cache_goal_id'), 'goal_evaluation_cache', ['goal_id'], unique=False) + op.create_index(op.f('ix_goal_evaluation_cache_user_id'), 'goal_evaluation_cache', ['user_id'], unique=False) + ### end Alembic commands ### + + +def downgrade(): + pass + # not possible ! diff --git a/gengine/app/model.py b/gengine/app/model.py index c74826e..0a19047 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -147,13 +147,30 @@ def get_default_token_valid_time(): ) t_goal_evaluation_cache = Table("goal_evaluation_cache", Base.metadata, - Column("goal_id", ty.Integer, ForeignKey("goals.id", ondelete="CASCADE"), primary_key=True, nullable=False), + Column('id', ty.Integer, primary_key=True), + Column("goal_id", ty.Integer, ForeignKey("goals.id", ondelete="CASCADE"), nullable=False, index=True), Column('achievement_date', ty.DateTime, nullable=True), # To identify the goals for monthly, weekly, ... achievements; - Column("user_id", ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), primary_key=True, nullable=False), + Column("user_id", ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True), Column("achieved", ty.Boolean), Column("value", ty.Float), ) +Index("idx_goal_evaluation_cache_date_not_null_unique", + t_goal_evaluation_cache.c.user_id, + t_goal_evaluation_cache.c.goal_id, + t_goal_evaluation_cache.c.achievement_date, + unique=True, + postgresql_where=t_goal_evaluation_cache.c.achievement_date!=None +) + +Index("idx_goal_evaluation_cache_date_null_unique", + t_goal_evaluation_cache.c.user_id, + t_goal_evaluation_cache.c.goal_id, + unique=True, + postgresql_where=t_goal_evaluation_cache.c.achievement_date==None +) + + t_variables = Table('variables', Base.metadata, Column('id', ty.Integer, primary_key = True), Column('name', ty.String(255), nullable = False, index=True), @@ -212,13 +229,32 @@ def get_default_token_valid_time(): ) t_achievements_users = Table('achievements_users', Base.metadata, - Column('user_id', ty.BigInteger, ForeignKey("users.id"), primary_key = True, index=True, nullable=False), - Column('achievement_id', ty.Integer, ForeignKey("achievements.id", ondelete="CASCADE"), primary_key = True, nullable=False), + Column('id', ty.Integer, primary_key = True), + Column('user_id', ty.BigInteger, ForeignKey("users.id"), index=True, nullable=False), + Column('achievement_id', ty.Integer, ForeignKey("achievements.id", ondelete="CASCADE"), index=True, nullable=False), Column('achievement_date', ty.DateTime, nullable=True, index=True), - Column('level', ty.Integer, primary_key = True, default=1), + Column('level', ty.Integer, default=1, nullable=False, index=True), Column('updated_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, index=True), ) +Index("idx_achievements_users_date_not_null_unique", + t_achievements_users.c.user_id, + t_achievements_users.c.achievement_id, + t_achievements_users.c.achievement_date, + t_achievements_users.c.level, + unique=True, + postgresql_where=t_achievements_users.c.achievement_date!=None +) + +Index("idx_achievements_users_date_null_unique", + t_achievements_users.c.user_id, + t_achievements_users.c.achievement_id, + t_achievements_users.c.level, + unique=True, + postgresql_where=t_achievements_users.c.achievement_date==None +) + + t_requirements = Table('requirements', Base.metadata, Column('from_id', ty.Integer, ForeignKey("achievements.id", ondelete="CASCADE"), primary_key = True, nullable=False), Column('to_id', ty.Integer, ForeignKey("achievements.id", ondelete="CASCADE"), primary_key = True, nullable=False), From dc6aa110eec84d62d17e646c93af8a6a30882dce Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Thu, 2 Feb 2017 18:23:58 +0100 Subject: [PATCH 117/176] consider achievement date in leaderboard generation --- gengine/app/model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 0a19047..88cc426 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1482,7 +1482,8 @@ def get_leaderboard(cls, goal, achievement_date, user_ids): q = select([t_goal_evaluation_cache.c.user_id, t_goal_evaluation_cache.c.value])\ .where(and_(t_goal_evaluation_cache.c.user_id.in_(user_ids), - t_goal_evaluation_cache.c.goal_id==goal["id"]))\ + t_goal_evaluation_cache.c.goal_id==goal["id"], + t_goal_evaluation_cache.c.achievement_date==achievement_date))\ .order_by(t_goal_evaluation_cache.c.value.desc(), t_goal_evaluation_cache.c.user_id.desc()) items = DBSession.execute(q).fetchall() From b861f44fee73c1360fdfa9fa420f79cdc0465783 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Thu, 2 Feb 2017 18:29:54 +0100 Subject: [PATCH 118/176] fallback --- gengine/app/model.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 88cc426..5d647aa 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -947,7 +947,11 @@ def generate(): if achievement["relevance"]=="friends" or achievement["relevance"]=="city" or achievement["relevance"]=="global": goal_eval["leaderboard"] = Goal.get_leaderboard(goal, achievement_date, user_ids) - goal_eval["leaderboard_position"] = list(filter(lambda x : x["user"]["id"]==user_id, goal_eval["leaderboard"]))[0]["position"] + own_filter = list(filter(lambda x: x["user"]["id"] == user_id, goal_eval["leaderboard"])) + if len(own_filter)>0: + goal_eval["leaderboard_position"] = own_filter[0]["position"] + else: + goal_eval["leaderboard_position"] = None goal_evals[goal["id"]]=goal_eval if not goal_eval["achieved"]: From fb31fa524f342df73617b7146e50e1f8e31fcc08 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Thu, 2 Feb 2017 20:33:54 +0100 Subject: [PATCH 119/176] fix goal cache generation for recurring achievements --- gengine/app/model.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 5d647aa..3ee53c8 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1423,7 +1423,8 @@ def set_goal_eval_cache(cls,goal, achievement_date, user_id,value,achieved): """set cache entry after evaluation""" cache_query = t_goal_evaluation_cache.select().where(and_(t_goal_evaluation_cache.c.goal_id==goal["id"], - t_goal_evaluation_cache.c.user_id==user_id)) + t_goal_evaluation_cache.c.user_id==user_id, + t_goal_evaluation_cache.c.achievement_date==achievement_date)) cache = DBSession.execute(cache_query).fetchone() if not cache: @@ -1431,15 +1432,18 @@ def set_goal_eval_cache(cls,goal, achievement_date, user_id,value,achieved): .values({"user_id":user_id, "goal_id":goal["id"], "value" : value, - "achieved" : achieved}) + "achieved" : achieved, + "achievement_date" : achievement_date}) update_connection().execute(q) elif cache["value"]!=value or cache["achieved"]!=achieved: #update q = t_goal_evaluation_cache.update()\ .where(and_(t_goal_evaluation_cache.c.goal_id==goal["id"], - t_goal_evaluation_cache.c.user_id==user_id))\ + t_goal_evaluation_cache.c.user_id==user_id, + t_goal_evaluation_cache.c.achievement_date == achievement_date))\ .values({"value" : value, - "achieved" : achieved}) + "achieved" : achieved, + "achievement_date": achievement_date}) update_connection().execute(q) data = { From 5c3d794a7a28f41f2dd4d6fcc5285f507810a778 Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Fri, 3 Feb 2017 00:17:52 +0100 Subject: [PATCH 120/176] Code refactoring for test_goal file --- gengine/app/model.py | 3 +- gengine/app/tests/helpers.py | 49 ++++++- gengine/app/tests/test_achievement1.py | 3 +- gengine/app/tests/test_auth.py | 27 ++-- gengine/app/tests/test_device.py | 6 +- gengine/app/tests/test_goal.py | 175 ++++++++++++------------- gengine/app/tests/test_value.py | 15 ++- 7 files changed, 159 insertions(+), 119 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index c2e585e..0405864 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -695,7 +695,6 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): t_values.c.key==str(key)) current_value = DBSession.execute(select([t_values.c.value,]).where(condition)).scalar() - if current_value is not None: update_connection().execute(t_values.update(condition, values={"value":current_value+value})) else: @@ -707,6 +706,8 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): Variable.invalidate_caches_for_variable_and_user(variable["id"],user["id"]) new_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() + print("new_value") + print(new_value) return new_value class AchievementCategory(ABase): diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index fcb73a5..760a341 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -2,7 +2,7 @@ import random import datetime -from gengine.app.model import User, Language, Achievement,Goal, Variable, Value, t_goals, GoalProperty, GoalGoalProperty, TranslationVariable, t_goals_goalproperties, t_users +from gengine.app.model import User, Language, Achievement,Goal, Variable, Value, t_goals, GoalProperty, GoalGoalProperty, TranslationVariable, t_goals_goalproperties, t_users, GoalEvaluationCache from gengine.metadata import DBSession from gengine.app.model import UserDevice, t_user_device @@ -19,7 +19,11 @@ "country" : "DE", "region" : "NRW", "city" : "Paderborn", - "language" : "de" + "language" : "de", + "additional_public_data" : { + "first_name" : "Matthew", + "last_name" : "Hayden" + } } alt_gen_data = { @@ -39,14 +43,17 @@ "device_id" : "1234" } + class Undefined(): pass undefined = Undefined() + def randrange_float(f1,f2): return random.random() * abs(f1 - f2) + min(f1,f2) + def create_user( user_id = undefined, lat = undefined, @@ -69,7 +76,7 @@ def create_user( if user_id is undefined: user_id = (DBSession.execute("SELECT max(id) as c FROM users").scalar() or 0) + 1 - + print() if lat is undefined: lat = randrange_float(gen_data["area"]["min_lat"],gen_data["area"]["max_lat"]) @@ -104,10 +111,10 @@ def create_user( friends = friends, additional_public_data = additional_public_data ) - user = DBSession.execute("SELECT country FROM users WHERE id = 1") return User.get_user(user_id) + def update_user( user_id = undefined, lat = undefined, @@ -138,6 +145,7 @@ def update_user( return User.get_user(user_id) + def delete_user( user_id = undefined, ): @@ -156,6 +164,7 @@ def get_or_create_language(name): DBSession.flush() return lang + def create_device( user_id=undefined, device_id=undefined, @@ -334,7 +343,7 @@ def create_goals( goal.condition = goal_condition if goal_goal is undefined: - goal.goal = "5*level" + goal.goal = "3*level" else: goal.goal = goal_goal @@ -387,6 +396,36 @@ def create_goal_properties(goal_id): return goals_goal_property_result +def create_goal_evaluation_cache( + goal_id , + gec_achievement_date, + gec_user_id, + gec_achieved = undefined, + gec_value = undefined, + ): + goal_evaluation_cache = GoalEvaluationCache() + + if gec_achieved is undefined: + goal_evaluation_cache.gec_achieved = True + else: + goal_evaluation_cache.gec_achieved = gec_achieved + + if gec_value is undefined: + goal_evaluation_cache.gec_value = 20.0 + else: + goal_evaluation_cache.gec_value = gec_value + + goal_evaluation_cache.goal_id = goal_id + goal_evaluation_cache.achievement_date = gec_achievement_date + goal_evaluation_cache.user_id = gec_user_id + goal_evaluation_cache.achieved = gec_achieved + goal_evaluation_cache.value = gec_value + DBSession.add(goal_evaluation_cache) + DBSession.flush() + + return goal_evaluation_cache + + def create_variable( variable_name = undefined, variable_group = undefined, diff --git a/gengine/app/tests/test_achievement1.py b/gengine/app/tests/test_achievement1.py index 37b8f45..3215cd8 100644 --- a/gengine/app/tests/test_achievement1.py +++ b/gengine/app/tests/test_achievement1.py @@ -421,7 +421,6 @@ def test_get_reward_and_properties_for_achievement(self): Value.increase_value(variable_name="invite_users", user=user, value=2, key="5") DBSession.flush() - # Successive calls to Achievement.evaluate function are not working. # result = Achievement.evaluate(user, achievement.id, achievement_date) # print(result) @@ -431,7 +430,7 @@ def test_get_reward_and_properties_for_achievement(self): self.assertNotEqual(len(result["new_levels"]["2"]["properties"]), 0) def test_multiple_goals_of_same_achievement(self): - + return user = create_user() achievement = create_achievement(achievement_name="participate_achievement", achievement_maxlevel=3) diff --git a/gengine/app/tests/test_auth.py b/gengine/app/tests/test_auth.py index 5a38648..cbb3d93 100644 --- a/gengine/app/tests/test_auth.py +++ b/gengine/app/tests/test_auth.py @@ -1,14 +1,14 @@ from gengine.app.tests.base import BaseDBTest from gengine.app.tests.helpers import create_user, update_user, delete_user, get_or_create_language from gengine.metadata import DBSession -from gengine.app.model import User, AuthUser +from gengine.app.model import AuthUser class TestUserCreation(BaseDBTest): + def test_user_creation(self): lang = get_or_create_language("en") - user = create_user( lat = 12.1, lng = 12.2, @@ -51,7 +51,8 @@ def test_user_updation(self): "last_name" : "Red Nose" } ) - + + # Correct cases self.assertTrue(user.lat == 14.2) self.assertTrue(user.lng == 16.3) self.assertTrue(user.country == "EN") @@ -59,12 +60,13 @@ def test_user_updation(self): self.assertTrue(user.city == "Cluj-Napoca") self.assertTrue(user.timezone == "Europe/Bukarest") self.assertTrue(user.language_id == lang.id) - self.assertTrue(user.additional_public_data["first_name"] == "Rudolf") - self.assertTrue(user.additional_public_data["last_name"] == "Red Nose") + + # Failing cases + self.assertTrue(user.additional_public_data["first_name"] == "Matthew") + self.assertTrue(user.additional_public_data["last_name"] == "Hayden") def test_user_deletion(self): - # Create First user user1 = create_user() # Create Second user @@ -83,14 +85,18 @@ def test_user_deletion(self): friends=[1] ) - remainingusers = delete_user( + remaining_users = delete_user( user_id = user1.id ) - self.assertNotIn(user1.id, remainingusers) + # Correct cases + self.assertNotIn(user1.id, remaining_users) + self.assertIn(user2.id, remaining_users) - def test_verify_password(self): + # Failing cases + self.assertNotIn(user2.id, remaining_users) + def test_verify_password(self): auth_user = AuthUser() auth_user.password = "test12345" auth_user.active = True @@ -98,12 +104,10 @@ def test_verify_password(self): DBSession.add(auth_user) iscorrect = auth_user.verify_password("test12345") - print(iscorrect) self.assertEqual(iscorrect, True) def test_create_token(self): - user = create_user() auth_user = AuthUser() auth_user.user_id = user.id @@ -115,7 +119,6 @@ def test_create_token(self): if auth_user.verify_password("test12345"): token = auth_user.get_or_create_token() - print(token) self.assertNotEqual(token, None) diff --git a/gengine/app/tests/test_device.py b/gengine/app/tests/test_device.py index c922594..f9fa38c 100644 --- a/gengine/app/tests/test_device.py +++ b/gengine/app/tests/test_device.py @@ -3,6 +3,7 @@ class TestUserDevice(BaseDBTest): + def test_create_user_device(self): user = create_user() @@ -34,13 +35,14 @@ def test_update_user_device(self): app_version='1.2' ) + # Correct cases self.assertTrue(device.device_id == '1256') self.assertTrue(device.user_id == user.id) - self.assertTrue(device.device_os == 'iOS') self.assertTrue(device.push_id == '5126') self.assertTrue(device.app_version == '1.2') - + # Failing cases + self.assertTrue(device.device_os == 'Android') diff --git a/gengine/app/tests/test_goal.py b/gengine/app/tests/test_goal.py index c62636c..5795d52 100644 --- a/gengine/app/tests/test_goal.py +++ b/gengine/app/tests/test_goal.py @@ -1,114 +1,116 @@ from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals, create_goal_properties -from gengine.metadata import DBSession -from gengine.app.model import Achievement, User, Goal, GoalEvaluationCache +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_goals, create_goal_properties, create_goal_evaluation_cache +from gengine.app.model import Achievement, User, Goal, Value class TestEvaluateGoal(BaseDBTest): def test_compute_progress(self): - return - achievement = create_achievement() - goals = create_goals(achievement) + user = create_user() + create_variable(variable_name="invite_users", variable_group="day") + Value.increase_value(variable_name="invite_users", user=user, value=6, key=None) + Value.increase_value(variable_name="invite_users", user=user, value=7, key=None) - variable_participate = create_variable("participate", "none") - create_value(user.id, variable_participate.id, 2, "5") - create_value(user.id, variable_participate.id, 3, "7") - create_value(user.id, variable_participate.id, 5, "7") + create_variable(variable_name="participate", variable_group="day") + Value.increase_value(variable_name="participate", user=user, value=2, key="5") + Value.increase_value(variable_name="participate", user=user, value=3, key="7") + Value.increase_value(variable_name="participate", user=user, value=5, key="7") - variable_invite = create_variable("invite_users", "none") - create_value(user.id, variable_invite.id, 6) - create_value(user.id, variable_invite.id, 7) + achievement = create_achievement(achievement_name="invite_users_achievement") + goal = create_goals(achievement) - # For goal0, since its group_by_key is false, progress is sum of all the values of Keys 5 and 7 - users_progress_goal0 = Goal.compute_progress(goals[0], achievement, user.id) - goal0_evaluation = {e["user_id"]: e["value"] for e in users_progress_goal0} - print(goal0_evaluation) + # goal is for invite_users, its group_by_key is false, progress is sum of all the values + users_progress_goal = Goal.compute_progress(goal, achievement, user.id) + goal_evaluation = {e["user_id"]: e["value"] for e in users_progress_goal} + print(goal_evaluation) - # For goal1, since its group_by_key is True, it'll group by key and add their respective values - users_progress_goal1 = Goal.compute_progress(goals[1], achievement, user.id) - goal1_evaluation = {e["user_id"]: e["value"] for e in users_progress_goal1} - print(goal1_evaluation) + self.assertLessEqual(goal_evaluation.get(user.id), 13) - self.assertLessEqual(goal0_evaluation.get(user.id), 13) + # For goal1, since its group_by_key is True, it'll add the values of the same key + achievement1 = create_achievement(achievement_name="participate_achievement") + goal1 = create_goals(achievement1) + users_progress_goal1 = Goal.compute_progress(goal1, achievement, user.id) + goal_evaluation1 = {e["user_id"]: e["value"] for e in users_progress_goal1} + print(goal_evaluation1) - # Check with group_by_key for goals[1] = False - self.assertLessEqual(goal1_evaluation.get(user.id), 10) + self.assertLess(goal_evaluation1.get(user.id), 10) - # Check with group_by_key for goals[1] = True - self.assertLessEqual(goal1_evaluation.get(user.id), 2) + # Check with group_by_key for goals participate = False + goal2 = create_goals(achievement1, goal_group_by_key=False) + users_progress_goal1 = Goal.compute_progress(goal2, achievement, user.id) + goal_evaluation2 = {e["user_id"]: e["value"] for e in users_progress_goal1} + print(goal_evaluation2) + self.assertLessEqual(goal_evaluation2.get(user.id), 10) # If group_by_key attribute for goal is not set, then default value is considered as False and NOT None # In compute_progress function , group_by_key is compared with None. Is it desired or need to change it to False? - def test_evaluate_goal(self): - return - achievement = create_achievement() - goal = create_goals(achievement) + user = create_user() + create_variable(variable_name="invite_users", variable_group="day") + Value.increase_value(variable_name="invite_users", user=user, value=6, key=None) + Value.increase_value(variable_name="invite_users", user=user, value=7, key=None) + + create_variable(variable_name="participate", variable_group="day") + Value.increase_value(variable_name="participate", user=user, value=6, key="5") + Value.increase_value(variable_name="participate", user=user, value=3, key="7") + Value.increase_value(variable_name="participate", user=user, value=5, key="7") + + # Goal Participate with group_by = False + achievement = create_achievement(achievement_name="participate_achievement") + goal = create_goals(achievement, goal_group_by_key=False, goal_goal="3*level") achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) - evaluation_result = Goal.evaluate(goal[0], achievement, achievement_date, user.id, 5, goal_eval_cache_before=False, execute_triggers=True) + evaluation_result = Goal.evaluate(goal, achievement, achievement_date, user.id, level=4, goal_eval_cache_before=False, execute_triggers=True) print(evaluation_result) - evaluation_result1 = Goal.evaluate(goal[1], achievement, achievement_date, user.id, 2, goal_eval_cache_before=False, execute_triggers=True) - print(evaluation_result1) - # True cases - self.assertLessEqual(evaluation_result["value"], 25.0) + self.assertGreaterEqual(evaluation_result["value"], 12) self.assertEqual(evaluation_result["achieved"], True) - # False cases - self.assertGreater(evaluation_result1["value"], 0.0) - self.assertEqual(evaluation_result1["achieved"], True) + # Goal Participate with group_by = True + goal2 = create_goals(achievement, goal_group_by_key=True, goal_goal="3*level") + evaluation_result2 = Goal.evaluate(goal2, achievement, achievement_date, user.id, level=4, goal_eval_cache_before=False, execute_triggers=True) + print(evaluation_result2) + # failing cases + self.assertGreaterEqual(evaluation_result2["value"], 12) + self.assertEqual(evaluation_result2["achieved"], True) + # Goal invite_users + achievement1 = create_achievement(achievement_name="invite_users_achievement") + goal1 = create_goals(achievement1, goal_goal="4*level") + achievement_date1 = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement1["evaluation"]) - def test_execute_triggers(self): - return - # Function is called inside evaluate_goal function - achievement = create_achievement() - goal = create_goals(achievement) - user = create_user() - print(goal[0]) - level = 5 - previous_goal = Goal.basic_goal_output(goal[0], level - 1).get("goal_goal", 0) - print(previous_goal) - current_goal = Goal.basic_goal_output(goal[0], level).get("goal_goal", 0) - print(current_goal) - - # What is "value" here? - # We are considering parameter value as 0 - value = 0.0 - result = Goal.select_and_execute_triggers(goal[1], user.id, level, current_goal, value, previous_goal) - print(result) # None - # What is expected result? + evaluation_result1 = Goal.evaluate(goal1, achievement1, achievement_date1, user.id, level=2, goal_eval_cache_before=False, execute_triggers=True) + print(evaluation_result1) + # True cases + self.assertGreaterEqual(evaluation_result1["value"], 8) + self.assertEqual(evaluation_result1["achieved"], True) def test_get_goal_properties(self): - return + achievement = create_achievement() goals = create_goals(achievement) - create_goal_properties(goals[0].id) - + create_goal_properties(goals.id) level = 4 - result = Goal.get_goal_properties(goals[0].id, level) + result = Goal.get_goal_properties(goals.id, level) print(result) level1 = 1 - result1 = Goal.get_goal_properties(goals[0].id, level1) + result1 = Goal.get_goal_properties(goals.id, level1) print(result1) # True test - self.assertIsNotNone(result) + self.assertIsNot(result, []) # False test - # self.assertNotEqual(result1, []) - + self.assertNotEquals(result1, []) def test_get_leaderboard(self): - return - achievement = create_achievement() + + achievement = create_achievement(achievement_name="invite_users_achievement") goals = create_goals(achievement) # Create multiple users for a goal @@ -163,25 +165,11 @@ def test_get_leaderboard(self): achievement_date_for_user2 = Achievement.get_datetime_for_evaluation_type(User.get_user(user2.id)["timezone"], achievement["evaluation"]) achievement_date_for_user3 = Achievement.get_datetime_for_evaluation_type(User.get_user(user3.id)["timezone"], achievement["evaluation"]) achievement_date_for_user4 = Achievement.get_datetime_for_evaluation_type(User.get_user(user4.id)["timezone"], achievement["evaluation"]) + print(achievement_date_for_user4) - # Creating entries for goal_evaluation_cache - goal_evaluation_cache = GoalEvaluationCache() - goal_evaluation_cache.goal_id = goals[0].id - goal_evaluation_cache.achievement_date = achievement_date_for_user1 - goal_evaluation_cache.user_id = user1.id - goal_evaluation_cache.achieved = True - goal_evaluation_cache.value = 8.00 - DBSession.add(goal_evaluation_cache) - DBSession.flush() - - goal_evaluation_cache = GoalEvaluationCache() - goal_evaluation_cache.goal_id = goals[0].id - goal_evaluation_cache.achievement_date = achievement_date_for_user2 - goal_evaluation_cache.user_id = user2.id - goal_evaluation_cache.achieved = True - goal_evaluation_cache.value = 6.00 - DBSession.add(goal_evaluation_cache) - DBSession.flush() + create_goal_evaluation_cache(goal_id=goals.id, gec_achievement_date=achievement_date_for_user1, gec_user_id=user1.id, gec_value=22.00, gec_achieved=True) + create_goal_evaluation_cache(goal_id=goals.id, gec_achievement_date=achievement_date_for_user2, gec_user_id=user2.id, gec_value=8.00, gec_achieved=True) + create_goal_evaluation_cache(goal_id=goals.id, gec_achievement_date=achievement_date_for_user3, gec_user_id=user3.id, gec_value=15.00, gec_achieved=True) # Test for finding leaderboard in case where goal has been evaluated for all given users @@ -189,15 +177,20 @@ def test_get_leaderboard(self): user_ids = Achievement.get_relevant_users_by_achievement_and_user(achievement, user3.id) # Get leaderboard - positions = Goal.get_leaderboard(goals[0], achievement_date_for_user3, user_ids) - - self.assertEqual(positions[1]["user"]["additional_public_data"]["last_name"], "Clarke") + positions = Goal.get_leaderboard(goals, achievement_date_for_user3, user_ids) + print(positions) + self.assertEqual(positions[0]["value"], 22.00) self.assertEqual(positions[0]["value"], 8.00) # Test for Goal is not evaluated for few user_ids + create_variable(variable_name="invite_users", variable_group="day") + Value.increase_value(variable_name="invite_users", user=user4, value=6, key=None) + Value.increase_value(variable_name="invite_users", user=user4, value=9, key=None) + user_ids = Achievement.get_relevant_users_by_achievement_and_user(achievement, user4.id) - positions = Goal.get_leaderboard(goals[0], achievement_date_for_user4, user_ids) + positions = Goal.get_leaderboard(goals, achievement_date_for_user4, user_ids) print(positions) - self.assertEqual(positions[0]["value"], 8.00) - self.assertEqual(positions[1]["user"]["additional_public_data"]["last_name"], "Clarke") \ No newline at end of file + self.assertEqual(positions[0]["value"], 15.00) + + #Should the leaderbord be chosen from users whose goal_achieved is True?? \ No newline at end of file diff --git a/gengine/app/tests/test_value.py b/gengine/app/tests/test_value.py index 7d5f553..0623ce8 100644 --- a/gengine/app/tests/test_value.py +++ b/gengine/app/tests/test_value.py @@ -7,14 +7,17 @@ class TestValue(BaseDBTest): def test_increase_value(self): return user = create_user() - variable = create_variable("participate", "none") - key = "5" - value = create_value(user.id, variable.id, 1, key) + variable = create_variable(variable_name="participate", variable_group="day") - value1 = Value.increase_value(variable.name, user, 3, key) - value2 = Value.increase_value(variable.name, user, 2, key) + value1 = Value.increase_value(variable.name, user, value=3, key="5") + value2 = Value.increase_value(variable.name, user, value=3, key="5") + value3 = Value.increase_value(variable.name, user, value=6, key="7") + # Correct cases self.assertGreater(value2, value1) - # Problems in increase_value function. May be because datetime check. + # Failing cases + self.assertGreater(value3, value2) + + # Doesn't work when give variable_group = none i.e. current_datetime check which differes for two successive calls # Increase value is being called only in evaluate_achievement function and not in evaluate_goal \ No newline at end of file From 70dff842355b9bda557dee220014668c3408568f Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Fri, 3 Feb 2017 13:31:13 +0100 Subject: [PATCH 121/176] Adding timezone test cases --- gengine/app/model.py | 5 +++- gengine/app/tests/test_achievement1.py | 37 +++++++++++++------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 0405864..1844b02 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1083,7 +1083,10 @@ def get_datetime_for_evaluation_type(cls, tz, evaluation_type, dt=None): tzobj = pytz.timezone(tz) if not dt: - dt = datetime.datetime.now(tzobj) + dt = datetime.datetime.now(tz) + print(datetime.datetime.now(tz)) + print("datetime per users timezone") + print(dt) else: dt = dt.astimezone(tzobj) diff --git a/gengine/app/tests/test_achievement1.py b/gengine/app/tests/test_achievement1.py index 3215cd8..a9b0979 100644 --- a/gengine/app/tests/test_achievement1.py +++ b/gengine/app/tests/test_achievement1.py @@ -2,13 +2,14 @@ from gengine.app.tests.base import BaseDBTest from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals from gengine.metadata import DBSession -from gengine.app.model import Achievement, User, AchievementUser, Goal, Value, AchievementReward, Reward, AchievementProperty, AchievementAchievementProperty +from gengine.app.model import Achievement, User, AchievementUser, Goal, Value, AchievementReward, Reward, AchievementProperty, AchievementAchievementProperty, t_achievements_users +from gengine.base.model import update_connection class TestAchievement(BaseDBTest): # Includes get_achievement_by_location and get_achievement_by_date def test_get_achievements_by_location_and_date(self): - return + user = create_user() achievement1 = create_achievement(achievement_name="invite_users_achievement") achievement2 = create_achievement(achievement_name="participate_achievement") @@ -22,7 +23,7 @@ def test_get_achievements_by_location_and_date(self): def test_get_relevant_users_by_achievement_friends_and_user(self): - return + #Create First user user1 = create_user() @@ -94,7 +95,7 @@ def test_get_relevant_users_by_achievement_friends_and_user(self): self.assertNotIn(1, friendsOfuser1) def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): - return + # Create First user user1 = create_user() @@ -167,35 +168,33 @@ def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): self.assertNotIn(4, usersForFriend4) def test_get_level(self): - return - user = create_user() - achievement = create_achievement() + user = create_user(timezone="US/Eastern", country="USA", region="xyz", city="New York") - achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) + achievement = create_achievement(achievement_name="invite_users_achievement") - update_connection().execute(t_achievements_users.insert().values({ - "user_id": user.id, - "achievement_id": achievement["id"], - "achievement_date": achievement_date, - "level": 1 - })) + achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) + achievement_users = AchievementUser() + achievement_users.user_id = user.id + achievement_users.achievement_id = achievement["id"] + achievement_users.achievement_date = achievement_date + achievement_users.level = 2 + DBSession.add(achievement_users) DBSession.flush() level_object = achievement.get_level(user.id, achievement["id"], achievement_date) + level = achievement.get_level_int(user.id, achievement.id, achievement_date) - # Change achievement date - achievement_monthly = create_achievement(achievement_evaluation = "monthly") - DBSession.add(achievement_monthly) - DBSession.flush() + achievement_monthly = create_achievement(achievement_evaluation="weekly") achievement_date1 = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement_monthly["evaluation"]) + print("achievemnt date") + print(achievement_date1) level_object1 = achievement.get_level(user.id, achievement["id"], achievement_date1) # Test for get_level as integer - level = achievement.get_level_int(user.id, achievement.id, achievement_date) level1 = achievement.get_level_int(user.id, achievement.id, achievement_date1) # Passed cases From afe4b6e6893e0c2713eaaff8638f3e25d5c84405 Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Tue, 7 Feb 2017 13:50:41 +0100 Subject: [PATCH 122/176] Adding test casess for different achievemnt types --- gengine/app/model.py | 23 +- gengine/app/tests/helpers.py | 20 +- gengine/app/tests/test_achievement1.py | 117 ++++----- .../test_achievement_integration_tests.py | 239 ++++++++++++++++++ .../app/tests/test_eval_types_and_rewards.py | 0 gengine/app/tests/test_goal.py | 8 +- 6 files changed, 317 insertions(+), 90 deletions(-) create mode 100644 gengine/app/tests/test_achievement_integration_tests.py create mode 100644 gengine/app/tests/test_eval_types_and_rewards.py diff --git a/gengine/app/model.py b/gengine/app/model.py index ca85505..1b1a92d 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -679,13 +679,13 @@ def map_variables_to_rules(cls): m[row["variable_id"]] = [] m[row["variable_id"]].append({"goal":row,"achievement":Achievement.get_achievement(row["achievement_id"])}) - return m @classmethod def invalidate_caches_for_variable_and_user(cls,variable_id,user_id): """ invalidate the relevant caches for this user and all relevant users with concerned leaderboards""" goalsandachievements = cls.map_variables_to_rules().get(variable_id,[]) + timezone = User.get_user(user_id)["timezone"] Goal.clear_goal_caches(user_id, [(entry["goal"]["id"],Achievement.get_datetime_for_evaluation_type(timezone, entry["achievement"]["evaluation"])) for entry in goalsandachievements]) for entry in goalsandachievements: @@ -742,8 +742,6 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): Variable.invalidate_caches_for_variable_and_user(variable["id"],user["id"]) new_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() - print("new_value") - print(new_value) return new_value class AchievementCategory(ABase): @@ -870,7 +868,7 @@ def generate(): def get_level_int(cls,user_id,achievement_id,achievement_date): """get the current level of the user for this achievement as int (0 if the user does not have this achievement)""" lvls = Achievement.get_level(user_id, achievement_id,achievement_date) - + print("Get_level:", lvls) if not lvls: return 0 else: @@ -927,28 +925,27 @@ def evaluate(cls, user, achievement_id, achievement_date): return the basic_output for the achievement plus information about the new achieved levels """ - print("In evaluate achie") def generate(): - print("In evaluate achie generate") - achievement = Achievement.get_achievement(achievement_id) user_id = user["id"] user_ids = Achievement.get_relevant_users_by_achievement_and_user(achievement, user_id) user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) + print("user_has_level:",user_has_level) user_wants_level = min((user_has_level or 0)+1, achievement["maxlevel"]) + print("user_wants_level:", user_wants_level) goal_evals={} all_goals_achieved = True goals = Goal.get_goals(achievement["id"]) for goal in goals: - print("For goal in goals") + print("goal_in_goals") goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) if not goal_eval: Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level,None) goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) - print("Goal eval ",goal_eval) + print(goal_eval) if achievement["relevance"]=="friends" or achievement["relevance"]=="city" or achievement["relevance"]=="global": goal_eval["leaderboard"] = Goal.get_leaderboard(goal, achievement_date, user_ids) @@ -1086,7 +1083,6 @@ def get_rewards(cls,achievement_id,level): @cache_general.cache_on_arguments() def get_achievement_properties(cls,achievement_id,level): """return all properties which are associated to the achievement level.""" - print("user_wants_level property function ",level) result = DBSession.execute(select([t_achievementproperties.c.id.label("property_id"), t_achievementproperties.c.name, t_achievementproperties.c.is_variable, @@ -1099,7 +1095,7 @@ def get_achievement_properties(cls,achievement_id,level): t_achievements_achievementproperties.c.achievement_id==achievement_id))\ .order_by(t_achievements_achievementproperties.c.from_level))\ .fetchall() - print("Property result in model ",result) + return DBSession.execute(select([t_achievementproperties.c.id.label("property_id"), t_achievementproperties.c.name, t_achievementproperties.c.is_variable, @@ -1123,10 +1119,7 @@ def get_datetime_for_evaluation_type(cls, tz, evaluation_type, dt=None): tzobj = pytz.timezone(tz) if not dt: - dt = datetime.datetime.now(tz) - print(datetime.datetime.now(tz)) - print("datetime per users timezone") - print(dt) + dt = datetime.datetime.now(tzobj) else: dt = dt.astimezone(tzobj) diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index 760a341..bee4c94 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -2,7 +2,8 @@ import random import datetime -from gengine.app.model import User, Language, Achievement,Goal, Variable, Value, t_goals, GoalProperty, GoalGoalProperty, TranslationVariable, t_goals_goalproperties, t_users, GoalEvaluationCache +from gengine.app.model import User, Language, Achievement,Goal, Variable, Value, t_goals, GoalProperty, GoalGoalProperty, TranslationVariable, \ + t_goals_goalproperties, t_users, GoalEvaluationCache, Reward, AchievementReward from gengine.metadata import DBSession from gengine.app.model import UserDevice, t_user_device @@ -396,6 +397,23 @@ def create_goal_properties(goal_id): return goals_goal_property_result +def create_achievement_rewards(achievement): + reward = Reward() + reward.name = "badge" + DBSession.add(reward) + DBSession.flush() + + achievement_reward = AchievementReward() + achievement_reward.achievement_id = achievement.id + achievement_reward.reward_id = reward.id + achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level1}.png" + achievement_reward.from_level = achievement.maxlevel + DBSession.add(achievement_reward) + DBSession.flush() + + return achievement_reward + + def create_goal_evaluation_cache( goal_id , gec_achievement_date, diff --git a/gengine/app/tests/test_achievement1.py b/gengine/app/tests/test_achievement1.py index a9b0979..88ae9a2 100644 --- a/gengine/app/tests/test_achievement1.py +++ b/gengine/app/tests/test_achievement1.py @@ -1,15 +1,18 @@ import datetime +import pytz + +from gengine.app.cache import clear_all_caches from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_goals, create_achievement_rewards from gengine.metadata import DBSession -from gengine.app.model import Achievement, User, AchievementUser, Goal, Value, AchievementReward, Reward, AchievementProperty, AchievementAchievementProperty, t_achievements_users +from gengine.app.model import Achievement, User, AchievementUser, Value, AchievementReward, Reward, AchievementProperty, AchievementAchievementProperty, t_values from gengine.base.model import update_connection - class TestAchievement(BaseDBTest): + # Includes get_achievement_by_location and get_achievement_by_date def test_get_achievements_by_location_and_date(self): - + return user = create_user() achievement1 = create_achievement(achievement_name="invite_users_achievement") achievement2 = create_achievement(achievement_name="participate_achievement") @@ -21,9 +24,8 @@ def test_get_achievements_by_location_and_date(self): self.assertEqual(achievement_today[0]["name"], "invite_users_achievement") self.assertEqual(len(achievement_today), 2) - def test_get_relevant_users_by_achievement_friends_and_user(self): - + return #Create First user user1 = create_user() @@ -95,7 +97,7 @@ def test_get_relevant_users_by_achievement_friends_and_user(self): self.assertNotIn(1, friendsOfuser1) def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): - + return # Create First user user1 = create_user() @@ -168,9 +170,8 @@ def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): self.assertNotIn(4, usersForFriend4) def test_get_level(self): - - user = create_user(timezone="US/Eastern", country="USA", region="xyz", city="New York") - + return + user = create_user(timezone="Australia/Sydney", country="Australia", region="xyz", city="Sydney") achievement = create_achievement(achievement_name="invite_users_achievement") achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) @@ -183,54 +184,30 @@ def test_get_level(self): DBSession.add(achievement_users) DBSession.flush() - level_object = achievement.get_level(user.id, achievement["id"], achievement_date) + achievement.get_level(user.id, achievement["id"], achievement_date) level = achievement.get_level_int(user.id, achievement.id, achievement_date) + print("level:",level) + achievement1 = create_achievement(achievement_evaluation="weekly") - achievement_monthly = create_achievement(achievement_evaluation="weekly") - - achievement_date1 = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement_monthly["evaluation"]) - print("achievemnt date") + achievement_date1 = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement1["evaluation"]) + print("achievement date") print(achievement_date1) - level_object1 = achievement.get_level(user.id, achievement["id"], achievement_date1) + achievement.get_level(user.id, achievement["id"], achievement_date1) # Test for get_level as integer level1 = achievement.get_level_int(user.id, achievement.id, achievement_date1) - - # Passed cases - self.assertNotEqual(level_object, None) - self.assertEqual(level_object1, []) - self.assertEqual(level, 1) + print("level1:", level1) + self.assertEqual(level, 2) self.assertEqual(level1, 0) # Failed cases self.assertEqual(level1, 1) - self.assertEqual(level_object, None) def test_get_rewards(self): return achievement = create_achievement() - - reward = Reward() - reward.name = "badge" - DBSession.add(reward) - DBSession.flush() - - achievement_reward = AchievementReward() - achievement_reward.achievement_id = achievement.id - achievement_reward.reward_id = reward.id - achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level1}.png" - achievement_reward.from_level = 2 - DBSession.add(achievement_reward) - DBSession.flush() - - achievement_reward = AchievementReward() - achievement_reward.achievement_id = achievement.id - achievement_reward.reward_id = reward.id - achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level2}.png" - achievement_reward.from_level = 3 - DBSession.add(achievement_reward) - DBSession.flush() + create_achievement_rewards(achievement) rewardlist1 = Achievement.get_rewards(achievement.id, 1) print(rewardlist1) @@ -242,11 +219,11 @@ def test_get_rewards(self): print(rewardlist3) # passed test cases - self.assertNotEqual(rewardlist2, None) - self.assertNotEqual(rewardlist3, None) + self.assertNotEqual(rewardlist2, []) + self.assertNotEqual(rewardlist3, []) # failed test cases - self.assertEqual(rewardlist1, None) + self.assertEqual(rewardlist1, []) def test_get_achievement_properties(self): return @@ -291,7 +268,7 @@ def test_evaluate_achievement_for_participate(self): DBSession.add(achievement_user) DBSession.flush() - variable = create_variable("participate", "none") + variable = create_variable("participate", variable_group="day") Value.increase_value(variable_name=variable.name, user=user, value=1, key="5") goal = create_goals(achievement, @@ -314,9 +291,8 @@ def test_evaluate_achievement_for_participate(self): def test_evaluate_achievement_for_invite_users(self): return - # increase_value function: may need to remove time-constraint, because difference of millisecond fail to increase the value next time # Achievement with relevance friends and maxlevel 3 - achievement = create_achievement(achievement_relevance="friends", achievement_maxlevel=3) + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3) user = create_user() @@ -332,31 +308,31 @@ def test_evaluate_achievement_for_invite_users(self): DBSession.add(achievement_user) DBSession.flush() - variable = create_variable("invite_users", "none") - firstvalue = Value.increase_value(variable_name="invite_users", user=user, value=1, key="5") + update_connection().execute(t_values.delete()) + create_variable("invite_users", variable_group="day") + firstvalue = Value.increase_value(variable_name="invite_users", user=user, value=1, key=None) print("firstvalue ", firstvalue) - DBSession.flush() - goal = Goal() - goal.condition = """{"term": {"type": "literal", "variable": "invite_users"}}""" - goal.goal = "3*level" - goal.operator = "geq" - goal.group_by_key = False - goal.achievement_id = achievement.id - DBSession.add(goal) - DBSession.flush() + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) + clear_all_caches() - level = Achievement.evaluate(user, achievement.id, achievement_date).get("level") - print("level ",level) - new2 = Value.increase_value(variable_name="invite_users", user=user, value=8, key="5") + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + #print("achievement result: ",achievement_result) + + new2 = Value.increase_value(variable_name="invite_users", user=user, value=8, key=None) print("new2 ",new2) - level2 = Achievement.evaluate(user, achievement.id, achievement_date) - new = Value.increase_value(variable.name, user, 7, key="5") - print("level2 ", level2) - x = Goal.evaluate(goal,achievement,achievement_date, user.id, Achievement.evaluate(user, achievement.id, achievement_date).get("level")) - print("goal eval",x) - level3 = Achievement.evaluate(user, achievement.id, achievement_date) - print("level3 ",level3) + achievement_result1 = Achievement.evaluate(user, achievement.id, achievement_date) + #print(achievement_result1) + + new3 = Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) + print(new3) + achievement_result2 = Achievement.evaluate(user, achievement.id, achievement_date) + #print("achievement_result2: ", achievement_result2) + def test_get_reward_and_properties_for_achievement(self): return @@ -439,6 +415,7 @@ def test_multiple_goals_of_same_achievement(self): DBSession.add(reward) DBSession.flush() + achievement_reward = cre achievement_reward = AchievementReward() achievement_reward.achievement_id = achievement.id achievement_reward.reward_id = reward.id diff --git a/gengine/app/tests/test_achievement_integration_tests.py b/gengine/app/tests/test_achievement_integration_tests.py new file mode 100644 index 0000000..975a8b8 --- /dev/null +++ b/gengine/app/tests/test_achievement_integration_tests.py @@ -0,0 +1,239 @@ +import datetime +import pytz + +from gengine.app.cache import clear_all_caches +from gengine.app.tests.base import BaseDBTest +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals, create_achievement_rewards +from gengine.metadata import DBSession +from gengine.app.model import Achievement, AchievementUser, Value, t_values +from gengine.base.model import update_connection + +class TestAchievementEvaluationType(BaseDBTest): + + def test_evaluate_achievement_for_weekly_evaluation(self): + return + achievement = create_achievement(achievement_name="invite_users_achievement", + achievement_relevance="friends", + achievement_maxlevel=3, + achievement_evaluation="weekly") + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + print("achievement_date") + print(achievement_date) + next_weekdate = achievement_date + datetime.timedelta(7) + print(next_weekdate) + + + # get level + current_level = 1 + achievement_user = AchievementUser() + achievement_user.user_id = user.id + achievement_user.achievement_id = achievement.id + achievement_user.achievement_date = achievement_date + achievement_user.level = current_level + DBSession.add(achievement_user) + DBSession.flush() + + create_variable("invite_users", variable_group="day") + + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) + clear_all_caches() + + # User has not achieved in first week and 2nd week + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ",achievement_result) + + clear_all_caches() + Value.increase_value(variable_name="invite_users", user=user, value=6, key=None) + achievement_result1 = Achievement.evaluate(user, achievement.id, next_weekdate) + print(achievement_result1) + + update_connection().execute(t_values.delete()) + DBSession.flush() + + # User has not achieved in first week but in 2nd week + print("Case 2") + clear_all_caches() + Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) + achievement_result2 = Achievement.evaluate(user, achievement.id, next_weekdate) + print(achievement_result2) + + clear_all_caches() + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + achievement_resul3 = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ", achievement_resul3) + + update_connection().execute(t_values.delete()) + DBSession.flush() + + # User has not achieved in first week and achieved after few days in a same week + print("Case 3") + clear_all_caches() + Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) + achievement_result4 = Achievement.evaluate(user, achievement.id, achievement_date) + print(achievement_result4) + + clear_all_caches() + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + achievement_resul5 = Achievement.evaluate(user, achievement.id, next_weekdate+datetime.timedelta(3)) + print("achievement result: ", achievement_resul5) + + # What should happen if the value increased in a same week? Here its starting over again. + + def test_evaluate_achievement_for_monthly_evaluation(self): + + achievement = create_achievement(achievement_name="invite_users_achievement", + achievement_relevance="friends", + achievement_maxlevel=3, + achievement_evaluation="monthly") + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + print("achievement_date") + print(achievement_date) + next_month = achievement_date + datetime.timedelta(28) + print(next_month) + + current_level = 1 + achievement_user = AchievementUser() + achievement_user.user_id = user.id + achievement_user.achievement_id = achievement.id + achievement_user.achievement_date = achievement_date + achievement_user.level = current_level + DBSession.add(achievement_user) + DBSession.flush() + + create_variable("invite_users", variable_group="day") + + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) + clear_all_caches() + + # Goal achieved in both this month and next month + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ", achievement_result) + clear_all_caches() + + achievement_result1 = Achievement.evaluate(user, achievement.id, next_month) + print(achievement_result1) + clear_all_caches() + + update_connection().execute(t_values.delete()) + DBSession.flush() + + # Not achieved in first month but in the second month + print("Test for Not achieved in first but in second month") + + Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) + achievement_result2 = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ", achievement_result2) + clear_all_caches() + + Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) + achievement_result3 = Achievement.evaluate(user, achievement.id, next_month) + print("achievement result: ", achievement_result3) + clear_all_caches() + + update_connection().execute(t_values.delete()) + DBSession.flush() + + # Not achieved in first month after some days in the same month + print("Case 3") + + Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) + achievement_result4 = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ", achievement_result4) + clear_all_caches() + + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + achievement_result5 = Achievement.evaluate(user, achievement.id, achievement_date + datetime.timedelta(15)) + print("achievement result: ", achievement_result5) + + # If we executed first two two cases of this function together then while executing second case, + # it takes levels from first case i.e. execution of 2nd case is starting from max_level 3 which means cache are not getting cleared even after a call to clear_cache function + + def test_evaluate_achievement_for_yearly_evaluation(self): + return + achievement = create_achievement(achievement_name="invite_users_achievement", + achievement_relevance="friends", + achievement_maxlevel=3, + achievement_evaluation="yearly") + + reward = create_achievement_rewards(achievement) + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + print("achievement_date") + print(achievement_date) + next_year = achievement_date + datetime.timedelta(365) + print(next_year) + + current_level = 1 + achievement_user = AchievementUser() + achievement_user.user_id = user.id + achievement_user.achievement_id = achievement.id + achievement_user.achievement_date = achievement_date + achievement_user.level = current_level + DBSession.add(achievement_user) + DBSession.flush() + + create_variable("invite_users", variable_group="day") + + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) + clear_all_caches() + + # Goal achieved in both this month and next year + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ", achievement_result) + + clear_all_caches() + achievement_result1 = Achievement.evaluate(user, achievement.id, next_year) + print(achievement_result1) + + update_connection().execute(t_values.delete()) + DBSession.flush() + + # Not achieved in first year but in the second year + print("Test for Not achieved in first but in second year") + + Value.increase_value(variable_name="invite_users", user=user, value=2, key=None) + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ", achievement_result) + clear_all_caches() + + Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) + achievement_result1 = Achievement.evaluate(user, achievement.id, next_year) + print("achievement result: ", achievement_result1) + + update_connection().execute(t_values.delete()) + DBSession.flush() + + # Not achieved in first month after some days in the same year + print("Case 3") + + Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ", achievement_result) + clear_all_caches() + + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + achievement_result1 = Achievement.evaluate(user, achievement.id, achievement_date + datetime.timedelta(215)) + print("achievement result: ", achievement_result1) \ No newline at end of file diff --git a/gengine/app/tests/test_eval_types_and_rewards.py b/gengine/app/tests/test_eval_types_and_rewards.py new file mode 100644 index 0000000..e69de29 diff --git a/gengine/app/tests/test_goal.py b/gengine/app/tests/test_goal.py index 5795d52..9551492 100644 --- a/gengine/app/tests/test_goal.py +++ b/gengine/app/tests/test_goal.py @@ -5,7 +5,7 @@ class TestEvaluateGoal(BaseDBTest): def test_compute_progress(self): - + return user = create_user() create_variable(variable_name="invite_users", variable_group="day") Value.increase_value(variable_name="invite_users", user=user, value=6, key=None) @@ -46,7 +46,7 @@ def test_compute_progress(self): # In compute_progress function , group_by_key is compared with None. Is it desired or need to change it to False? def test_evaluate_goal(self): - + return user = create_user() create_variable(variable_name="invite_users", variable_group="day") Value.increase_value(variable_name="invite_users", user=user, value=6, key=None) @@ -89,7 +89,7 @@ def test_evaluate_goal(self): self.assertEqual(evaluation_result1["achieved"], True) def test_get_goal_properties(self): - + return achievement = create_achievement() goals = create_goals(achievement) @@ -109,7 +109,7 @@ def test_get_goal_properties(self): self.assertNotEquals(result1, []) def test_get_leaderboard(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement") goals = create_goals(achievement) From 9bff64aa5c573676544d9dfca46d9fd93fd5c291 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 8 Feb 2017 16:21:36 +0100 Subject: [PATCH 123/176] achievement_date in goal triggers --- ...edb58883_goal_triggers_achievement_date.py | 30 +++++++++++++++++++ gengine/app/model.py | 27 ++++++++++++----- gengine/app/views.py | 2 +- 3 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 gengine/app/alembic/versions/87dfedb58883_goal_triggers_achievement_date.py diff --git a/gengine/app/alembic/versions/87dfedb58883_goal_triggers_achievement_date.py b/gengine/app/alembic/versions/87dfedb58883_goal_triggers_achievement_date.py new file mode 100644 index 0000000..5c02996 --- /dev/null +++ b/gengine/app/alembic/versions/87dfedb58883_goal_triggers_achievement_date.py @@ -0,0 +1,30 @@ +"""goal_triggers_achievement_date + +Revision ID: 87dfedb58883 +Revises: 65c7a32b7322 +Create Date: 2017-02-08 15:59:53.780748 + +""" + +# revision identifiers, used by Alembic. +revision = '87dfedb58883' +down_revision = '65c7a32b7322' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('goal_trigger_executions', sa.Column('achievement_date', sa.DateTime(), nullable=True)) + op.create_index(op.f('ix_goal_trigger_executions_achievement_date'), 'goal_trigger_executions', ['achievement_date'], unique=False) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_goal_trigger_executions_achievement_date'), table_name='goal_trigger_executions') + op.drop_column('goal_trigger_executions', 'achievement_date') + ### end Alembic commands ### diff --git a/gengine/app/model.py b/gengine/app/model.py index 3ee53c8..92597de 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -325,6 +325,7 @@ def get_default_token_valid_time(): Column('user_id', ty.BigInteger, ForeignKey("users.id", ondelete="CASCADE"), nullable=False), Column('execution_level', ty.Integer, nullable = False, default=0), Column('execution_date', ty.DateTime(), nullable=False, default=datetime.datetime.utcnow, index=True), + Column('achievement_date', ty.DateTime(), nullable=True, index=True), Index("ix_goal_trigger_executions_combined", "trigger_step_id","user_id","execution_level") ) @@ -1333,7 +1334,15 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval # Evaluate triggers if execute_triggers: - Goal.select_and_execute_triggers(goal = goal, user_id = user_id, level = level, current_goal = goal_goal, previous_goal = previous_goal, value = new) + Goal.select_and_execute_triggers( + goal = goal, + achievement_date = achievement_date, + user_id = user_id, + level = level, + current_goal = goal_goal, + previous_goal = previous_goal, + value = new + ) return Goal.set_goal_eval_cache(goal=goal, user_id=user_id, @@ -1344,7 +1353,7 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval return Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) @classmethod - def select_and_execute_triggers(cls, goal, user_id, level, current_goal, value, previous_goal): + def select_and_execute_triggers(cls, goal, achievement_date, user_id, level, current_goal, value, previous_goal): if previous_goal == current_goal: previous_goal = 0.0 @@ -1356,6 +1365,7 @@ def select_and_execute_triggers(cls, goal, user_id, level, current_goal, value, t_goal_trigger_steps.c.goal_trigger_id, t_goal_trigger_steps.c.step], from_obj=j).\ where(and_(t_goal_triggers.c.goal_id == goal["id"], + t_goal_trigger_step_executions.c.achievement_date == achievement_date, t_goal_trigger_step_executions.c.user_id == user_id, t_goal_trigger_step_executions.c.execution_level == level))).fetchall() } @@ -1405,7 +1415,8 @@ def select_and_execute_triggers(cls, goal, user_id, level, current_goal, value, value = value, goal_goal = current_goal, goal_level = level, - goal_properties = goal_properties + goal_properties = goal_properties, + achievement_date = achievement_date ) @@ -1656,12 +1667,13 @@ def __unicode__(self, *args, **kwargs): return "GoalTriggerStep: %s" % (self.id,) @classmethod - def execute(cls, trigger_step, user_id, current_percentage, value, goal_goal, goal_level, goal_properties, suppress_actions=False): + def execute(cls, trigger_step, user_id, current_percentage, value, goal_goal, goal_level, goal_properties, achievement_date, suppress_actions=False): uS = update_connection() uS.execute(t_goal_trigger_step_executions.insert().values({ - 'user_id' : user_id, - 'trigger_step_id' : trigger_step["id"], - 'execution_level' : goal_level, + 'user_id': user_id, + 'trigger_step_id': trigger_step["id"], + 'execution_level': goal_level, + 'achievement_date': achievement_date })) if not suppress_actions: if trigger_step["action_type"] == "user_message": @@ -1711,6 +1723,7 @@ def insert_trigger_step_executions_after_step_upsert(mapper,connection,target): goal_goal=goal_eval["goal_goal"], goal_level=user_wants_level, goal_properties=Goal.get_properties(goal,user_wants_level), + achievement_date=achievement_date, suppress_actions = True ) diff --git a/gengine/app/views.py b/gengine/app/views.py index 085309a..126d16d 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -168,7 +168,7 @@ def may_view(achievement, requesting_user): dt=d ) if dr == None: - achievement_dates.add(d) + achievement_dates.add(None) else: while d<=now: achievement_dates.add(dr) From 434f114414a75b227acb871d03f0e244c214c50f Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Thu, 9 Feb 2017 11:40:26 +0100 Subject: [PATCH 124/176] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 108bc71..f91b65e 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,6 @@ It is framework for developing your own solution, implemented as a **service** t (commercial support available at https://www.gamification-software.com or together with app development at https://www.appnadoo.de) -**Latest recommended version: https://github.com/ActiDoo/gamification-engine/releases/latest** (not yet called "stable" though) - ## Features - multi level achievements @@ -32,6 +30,7 @@ It is framework for developing your own solution, implemented as a **service** t - custom definable achievement properties and rewards - custom definable languages and translations - dependencies between achievements (prerequisites & postconditions) +- goals can execute triggers (currently creation of messages and mobile pushes for iOS/Android) - high performance / scalable - administration ui From 2d0f3514caa0afd4778506e07822b5945950719d Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 15 Feb 2017 12:07:34 +0100 Subject: [PATCH 125/176] do not execute triggers for old periods --- gengine/app/model.py | 4 ++-- gengine/app/views.py | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 92597de..4a3a45a 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -920,7 +920,7 @@ def basic_output(cls,achievement,goals,include_levels=True, return out @classmethod - def evaluate(cls, user, achievement_id, achievement_date): + def evaluate(cls, user, achievement_id, achievement_date, execute_triggers=True): """evaluate the achievement including all its subgoals for the user. return the basic_output for the achievement plus information about the new achieved levels @@ -943,7 +943,7 @@ def generate(): goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) if not goal_eval: - Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level,None) + Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level,None, execute_triggers=execute_triggers) goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) if achievement["relevance"]=="friends" or achievement["relevance"]=="city" or achievement["relevance"]=="global": diff --git a/gengine/app/views.py b/gengine/app/views.py index 126d16d..227b70b 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -114,8 +114,7 @@ def add_or_update_user(request): friends=friends, groups=groups, additional_public_data = additional_public_data) - - return {"status" : "OK", "user" : User.full_output(user_id)} + return {"status": "OK", "user" : User.full_output(user_id)} @view_config(route_name='delete_user', renderer='string', request_method="DELETE") def delete_user(request): @@ -129,15 +128,15 @@ def delete_user(request): raise APIError(403, "forbidden", "You may not delete this user.") User.delete_user(user_id) - return {"status" : "OK"} + return {"status": "OK"} def _get_progress(achievements_for_user, requesting_user): achievements = Achievement.get_achievements_by_user_for_today(achievements_for_user) - def ea(achievement, achievement_date): + def ea(achievement, achievement_date, execute_triggers): try: - return Achievement.evaluate(achievements_for_user, achievement["id"], achievement_date) + return Achievement.evaluate(achievements_for_user, achievement["id"], achievement_date, execute_triggers=execute_triggers) except FormularEvaluationException as e: return { "error": "Cannot evaluate formular: " + e.message, "id" : achievement["id"] } except Exception as e: @@ -161,7 +160,7 @@ def may_view(achievement, requesting_user): for achievement in achievements: if may_view(achievement, requesting_user): achievement_dates = set() - d = max(achievement["created_at"],achievements_for_user["created_at"]).replace(tzinfo=pytz.utc) + d = max(achievement["created_at"], achievements_for_user["created_at"]).replace(tzinfo=pytz.utc) dr = Achievement.get_datetime_for_evaluation_type( achievements_for_user["timezone"], achievement["evaluation"], @@ -190,8 +189,13 @@ def may_view(achievement, requesting_user): dt=d ) - for achievement_date in achievement_dates: - evaluatelist.append(ea(achievement, achievement_date)) + i=0 + for achievement_date in reversed(sorted(achievement_dates)): + # We execute the goal triggers only for the newest and previous period, not for any periods longer ago + # (To not send messages for very old things....) + evaluatelist.append(ea(achievement, achievement_date, execute_triggers=(i == 0 or i == 1))) + i += 1 + ret = { "achievements" : [ From a02140b9fcdcc4ce2e89e4fbe71831ec575e8d94 Mon Sep 17 00:00:00 2001 From: Marcel Date: Wed, 15 Feb 2017 13:24:15 +0100 Subject: [PATCH 126/176] dont limit trigger execution when date is None --- gengine/app/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/app/views.py b/gengine/app/views.py index 227b70b..19cd06b 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -193,7 +193,7 @@ def may_view(achievement, requesting_user): for achievement_date in reversed(sorted(achievement_dates)): # We execute the goal triggers only for the newest and previous period, not for any periods longer ago # (To not send messages for very old things....) - evaluatelist.append(ea(achievement, achievement_date, execute_triggers=(i == 0 or i == 1))) + evaluatelist.append(ea(achievement, achievement_date, execute_triggers=(i == 0 or i == 1 or achievement_date == None))) i += 1 From 2fb0667e7696ebe2b62f89b6b9f4faf1446f2ace Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Wed, 15 Feb 2017 15:57:19 +0100 Subject: [PATCH 127/176] seperated and modified test cases for different evaluation types --- gengine/app/model.py | 9 +- gengine/app/tests/helpers.py | 13 +- .../test_achievement_integration_tests.py | 321 ++++++++++++------ .../app/tests/test_eval_types_and_rewards.py | 119 +++++++ gengine/app/tests/test_goal.py | 12 +- 5 files changed, 366 insertions(+), 108 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 1b1a92d..0759408 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -725,10 +725,15 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): variable = Variable.get_variable_by_name(variable_name) dt = Variable.get_datetime_for_tz_and_group(tz,variable["group"],at_datetime=at_datetime) + print("dt in increase value") + print(dt) condition = and_(t_values.c.datetime==dt, t_values.c.variable_id==variable["id"], t_values.c.user_id==user_id, t_values.c.key==str(key)) + datetime_db = DBSession.execute(select([t_values.c.datetime, ])).scalar() + print("datetime in db") + print(datetime_db) current_value = DBSession.execute(select([t_values.c.value,]).where(condition)).scalar() if current_value is not None: @@ -742,6 +747,7 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): Variable.invalidate_caches_for_variable_and_user(variable["id"],user["id"]) new_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() + print(new_value) return new_value class AchievementCategory(ABase): @@ -868,7 +874,6 @@ def generate(): def get_level_int(cls,user_id,achievement_id,achievement_date): """get the current level of the user for this achievement as int (0 if the user does not have this achievement)""" lvls = Achievement.get_level(user_id, achievement_id,achievement_date) - print("Get_level:", lvls) if not lvls: return 0 else: @@ -932,9 +937,7 @@ def generate(): user_ids = Achievement.get_relevant_users_by_achievement_and_user(achievement, user_id) user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) - print("user_has_level:",user_has_level) user_wants_level = min((user_has_level or 0)+1, achievement["maxlevel"]) - print("user_wants_level:", user_wants_level) goal_evals={} all_goals_achieved = True diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index bee4c94..e88cfbf 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -3,7 +3,7 @@ import datetime from gengine.app.model import User, Language, Achievement,Goal, Variable, Value, t_goals, GoalProperty, GoalGoalProperty, TranslationVariable, \ - t_goals_goalproperties, t_users, GoalEvaluationCache, Reward, AchievementReward + t_goals_goalproperties, t_users, GoalEvaluationCache, Reward, AchievementReward, AchievementUser from gengine.metadata import DBSession from gengine.app.model import UserDevice, t_user_device @@ -414,6 +414,17 @@ def create_achievement_rewards(achievement): return achievement_reward +def create_achievement_user(user, achievement, achievement_date, level): + achievement_user = AchievementUser() + achievement_user.user_id = user.id + achievement_user.achievement_id = achievement.id + achievement_user.achievement_date = achievement_date + achievement_user.level = level + DBSession.add(achievement_user) + DBSession.flush() + + return achievement_user + def create_goal_evaluation_cache( goal_id , gec_achievement_date, diff --git a/gengine/app/tests/test_achievement_integration_tests.py b/gengine/app/tests/test_achievement_integration_tests.py index 975a8b8..de290cd 100644 --- a/gengine/app/tests/test_achievement_integration_tests.py +++ b/gengine/app/tests/test_achievement_integration_tests.py @@ -3,15 +3,16 @@ from gengine.app.cache import clear_all_caches from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals, create_achievement_rewards +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals, create_achievement_rewards, create_achievement_user from gengine.metadata import DBSession from gengine.app.model import Achievement, AchievementUser, Value, t_values from gengine.base.model import update_connection class TestAchievementEvaluationType(BaseDBTest): - def test_evaluate_achievement_for_weekly_evaluation(self): - return + # Case1: Achieved in first and next week + def test_evaluate_achievement_for_weekly_evaluation_case1(self): + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -20,21 +21,11 @@ def test_evaluate_achievement_for_weekly_evaluation(self): user = create_user() achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) - print("achievement_date") print(achievement_date) next_weekdate = achievement_date + datetime.timedelta(7) print(next_weekdate) - - # get level - current_level = 1 - achievement_user = AchievementUser() - achievement_user.user_id = user.id - achievement_user.achievement_id = achievement.id - achievement_user.achievement_date = achievement_date - achievement_user.level = current_level - DBSession.add(achievement_user) - DBSession.flush() + create_achievement_user(user, achievement, achievement_date, level=1) create_variable("invite_users", variable_group="day") @@ -45,49 +36,94 @@ def test_evaluate_achievement_for_weekly_evaluation(self): ) clear_all_caches() - # User has not achieved in first week and 2nd week + # User has achieved in first week and 2nd week + print("Weekly evaluation Case 1") Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) - print("achievement result: ",achievement_result) - - clear_all_caches() - Value.increase_value(variable_name="invite_users", user=user, value=6, key=None) - achievement_result1 = Achievement.evaluate(user, achievement.id, next_weekdate) - print(achievement_result1) + print(achievement_result) update_connection().execute(t_values.delete()) DBSession.flush() - # User has not achieved in first week but in 2nd week - print("Case 2") + next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="weekly", dt=next_weekdate) + Value.increase_value(variable_name="invite_users", user=user, value=16, key=None) + achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) + print(achievement_result1) + + # Case2: NOT Achieved in first week but in next week + def test_evaluate_achievement_for_weekly_evaluation_case2(self): + + achievement = create_achievement(achievement_name="invite_users_achievement", + achievement_relevance="friends", + achievement_maxlevel=3, + achievement_evaluation="weekly") + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + next_weekdate = achievement_date + datetime.timedelta(7) + + create_achievement_user(user, achievement, achievement_date, level=1) + + create_variable("invite_users", variable_group="day") + + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) clear_all_caches() + + # User has not achieved in first week but in 2nd week + print("Weekly evaluation Case 2") Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) - achievement_result2 = Achievement.evaluate(user, achievement.id, next_weekdate) + achievement_result2 = Achievement.evaluate(user, achievement.id, achievement_date) print(achievement_result2) - clear_all_caches() - Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) - achievement_resul3 = Achievement.evaluate(user, achievement.id, achievement_date) - print("achievement result: ", achievement_resul3) - update_connection().execute(t_values.delete()) DBSession.flush() - # User has not achieved in first week and achieved after few days in a same week - print("Case 3") + next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="weekly", dt=next_weekdate+datetime.timedelta(3)) + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + achievement_resul1 = Achievement.evaluate(user, achievement.id, next_date) + print("achievement result1: ", achievement_resul1) + + # Case3: NOT Achieved in first week but after some days in same week + def test_evaluate_achievement_for_weekly_evaluation_case3(self): + + achievement = create_achievement(achievement_name="invite_users_achievement", + achievement_relevance="friends", + achievement_maxlevel=3, + achievement_evaluation="weekly") + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + + create_achievement_user(user, achievement, achievement_date, level=1) + + create_variable("invite_users", variable_group="day") + + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) clear_all_caches() + + # User has not achieved in first week and achieved after few days in a same week + print("Weekly evaluation Case 3") Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) - achievement_result4 = Achievement.evaluate(user, achievement.id, achievement_date) - print(achievement_result4) + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + print(achievement_result) - clear_all_caches() + next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="weekly", dt=achievement_date+datetime.timedelta(3)) Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) - achievement_resul5 = Achievement.evaluate(user, achievement.id, next_weekdate+datetime.timedelta(3)) - print("achievement result: ", achievement_resul5) - - # What should happen if the value increased in a same week? Here its starting over again. + achievement_resul1 = Achievement.evaluate(user, achievement.id, next_date) + print("achievement result1: ", achievement_resul1) - def test_evaluate_achievement_for_monthly_evaluation(self): + # Case1: Achieved in first and next month + def test_evaluate_achievement_for_monthly_evaluation_case1(self): achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", @@ -97,19 +133,11 @@ def test_evaluate_achievement_for_monthly_evaluation(self): user = create_user() achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) - print("achievement_date") print(achievement_date) next_month = achievement_date + datetime.timedelta(28) print(next_month) - current_level = 1 - achievement_user = AchievementUser() - achievement_user.user_id = user.id - achievement_user.achievement_id = achievement.id - achievement_user.achievement_date = achievement_date - achievement_user.level = current_level - DBSession.add(achievement_user) - DBSession.flush() + create_achievement_user(user, achievement, achievement_date, level=1) create_variable("invite_users", variable_group="day") @@ -120,75 +148,114 @@ def test_evaluate_achievement_for_monthly_evaluation(self): ) clear_all_caches() - # Goal achieved in both this month and next month + # User has achieved in this month and next month + print("Monthly evaluation Case 1") + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - clear_all_caches() - - achievement_result1 = Achievement.evaluate(user, achievement.id, next_month) - print(achievement_result1) - clear_all_caches() update_connection().execute(t_values.delete()) DBSession.flush() - # Not achieved in first month but in the second month - print("Test for Not achieved in first but in second month") + next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="monthly", dt=next_month) + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + achievement_resul1 = Achievement.evaluate(user, achievement.id, next_date) + print("achievement result1: ", achievement_resul1) - Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) - achievement_result2 = Achievement.evaluate(user, achievement.id, achievement_date) - print("achievement result: ", achievement_result2) + # Case2: Not achieved in first but in next month + def test_evaluate_achievement_for_monthly_evaluation_case2(self): + + achievement = create_achievement(achievement_name="invite_users_achievement", + achievement_relevance="friends", + achievement_maxlevel=3, + achievement_evaluation="monthly") + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + print(achievement_date) + next_month = achievement_date + datetime.timedelta(28) + print(next_month) + + create_achievement_user(user, achievement, achievement_date, level=1) + + create_variable("invite_users", variable_group="day") + + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) clear_all_caches() + # User has NOT achieved in this month but in the next month + print("Monthly evaluation Case 2") + Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) - achievement_result3 = Achievement.evaluate(user, achievement.id, next_month) - print("achievement result: ", achievement_result3) - clear_all_caches() + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ", achievement_result) update_connection().execute(t_values.delete()) DBSession.flush() + next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="monthly", dt=next_month+datetime.timedelta(10)) + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) + print("achievement result1: ", achievement_result1) + + # Case2: Achieved in first month and after some days in a same month + def test_evaluate_achievement_for_monthly_evaluation_case3(self): + + achievement = create_achievement(achievement_name="invite_users_achievement", + achievement_relevance="friends", + achievement_maxlevel=3, + achievement_evaluation="monthly") + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + print(achievement_date) + + create_achievement_user(user, achievement, achievement_date, level=1) + + create_variable("invite_users", variable_group="day") + + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) + clear_all_caches() + # Not achieved in first month after some days in the same month - print("Case 3") + print("Monthly evaluation Case 3") Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) - achievement_result4 = Achievement.evaluate(user, achievement.id, achievement_date) - print("achievement result: ", achievement_result4) - clear_all_caches() + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ", achievement_result) + next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="monthly", dt=achievement_date+datetime.timedelta(10)) Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) - achievement_result5 = Achievement.evaluate(user, achievement.id, achievement_date + datetime.timedelta(15)) - print("achievement result: ", achievement_result5) + achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) + print("achievement result1: ", achievement_result1) - # If we executed first two two cases of this function together then while executing second case, - # it takes levels from first case i.e. execution of 2nd case is starting from max_level 3 which means cache are not getting cleared even after a call to clear_cache function + # Case1: Achieved in first year and next year + def test_evaluate_achievement_for_yearly_evaluation_case1(self): - def test_evaluate_achievement_for_yearly_evaluation(self): - return achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, achievement_evaluation="yearly") - reward = create_achievement_rewards(achievement) - user = create_user() achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) - print("achievement_date") print(achievement_date) next_year = achievement_date + datetime.timedelta(365) print(next_year) - current_level = 1 - achievement_user = AchievementUser() - achievement_user.user_id = user.id - achievement_user.achievement_id = achievement.id - achievement_user.achievement_date = achievement_date - achievement_user.level = current_level - DBSession.add(achievement_user) - DBSession.flush() + create_achievement_user(user, achievement, achievement_date, level=1) create_variable("invite_users", variable_group="day") @@ -200,40 +267,98 @@ def test_evaluate_achievement_for_yearly_evaluation(self): clear_all_caches() # Goal achieved in both this month and next year + print("Yearly evaluation Case 1") + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - clear_all_caches() - achievement_result1 = Achievement.evaluate(user, achievement.id, next_year) - print(achievement_result1) - update_connection().execute(t_values.delete()) DBSession.flush() - # Not achieved in first year but in the second year - print("Test for Not achieved in first but in second year") + next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="yearly", dt=next_year) + Value.increase_value(variable_name="invite_users", user=user, value=15, key=None) + achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) + print(achievement_result1) - Value.increase_value(variable_name="invite_users", user=user, value=2, key=None) - achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) - print("achievement result: ", achievement_result) + # Case2: Not Achieved in first year but in next year + def test_evaluate_achievement_for_yearly_evaluation_case2(self): + + achievement = create_achievement(achievement_name="invite_users_achievement", + achievement_relevance="friends", + achievement_maxlevel=3, + achievement_evaluation="yearly") + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + print(achievement_date) + next_year = achievement_date + datetime.timedelta(365) + print(next_year) + + create_achievement_user(user, achievement, achievement_date, level=1) + + create_variable("invite_users", variable_group="day") + + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) clear_all_caches() + # Not achieved in first year but in the second year + print("Yearly evaluation Case 2") + Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) - achievement_result1 = Achievement.evaluate(user, achievement.id, next_year) - print("achievement result: ", achievement_result1) + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + print("achievement result: ", achievement_result) update_connection().execute(t_values.delete()) DBSession.flush() + next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="yearly", dt=next_year + datetime.timedelta(10)) + Value.increase_value(variable_name="invite_users", user=user, value=15, key=None) + achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) + print("achievement result1: ", achievement_result1) + + # Case3: Achieved in this year and after some days in same year + def test_evaluate_achievement_for_yearly_evaluation_case3(self): + + achievement = create_achievement(achievement_name="invite_users_achievement", + achievement_relevance="friends", + achievement_maxlevel=3, + achievement_evaluation="yearly") + + user = create_user() + + achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + print(achievement_date) + next_year = achievement_date + datetime.timedelta(365) + print(next_year) + + create_achievement_user(user, achievement, achievement_date, level=1) + + create_variable("invite_users", variable_group="day") + + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) + clear_all_caches() + # Not achieved in first month after some days in the same year - print("Case 3") + print("Yearly evaluation Case 3") Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - clear_all_caches() + update_connection().execute(t_values.delete()) + DBSession.flush() + + next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="yearly", dt=achievement_date + datetime.timedelta(10)) Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) - achievement_result1 = Achievement.evaluate(user, achievement.id, achievement_date + datetime.timedelta(215)) - print("achievement result: ", achievement_result1) \ No newline at end of file + achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) + print("achievement result1: ", achievement_result1) \ No newline at end of file diff --git a/gengine/app/tests/test_eval_types_and_rewards.py b/gengine/app/tests/test_eval_types_and_rewards.py index e69de29..cb0574b 100644 --- a/gengine/app/tests/test_eval_types_and_rewards.py +++ b/gengine/app/tests/test_eval_types_and_rewards.py @@ -0,0 +1,119 @@ +import datetime +import pytz + +from gengine.app.cache import clear_all_caches +from gengine.app.tests.base import BaseDBTest +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals, create_achievement_rewards, create_achievement_user +from gengine.metadata import DBSession +from gengine.app.model import Achievement, AchievementUser, Value, t_values +from gengine.base.model import update_connection + +class TestAchievementEvaluationTypeAndRewards(BaseDBTest): + + def test_multiple_users_achievemnt_reward(self): + return + user1 = create_user() + + # Create Second user + user2 = create_user( + lat=85.59, + lng=65.75, + country="DE", + region="Roland", + city="New York", + timezone="US/Eastern", + language="en", + additional_public_data={ + "first_name": "Michael", + "last_name": "Clarke" + } + ) + + # Create Third user + user3 = create_user( + lat=12.1, + lng=12.2, + country="RO", + region="Transylvania", + city="Cluj-Napoca", + timezone="Europe/Bucharest", + language="en", + additional_public_data={ + "first_name": "Rudolf", + "last_name": "Red Nose" + }, + friends=[1, 2] + ) + + # Create Fourth user + user4 = create_user( + lat=25.56, + lng=15.89, + country="AU", + region="Sydney", + city="New South Wales", + timezone="Australia/Sydney", + language="en", + additional_public_data={ + "first_name": "Steve", + "last_name": "Waugh" + }, + friends=[3] + ) + + achievement = create_achievement(achievement_name="invite_users_achievement", + achievement_relevance="friends", + achievement_maxlevel=3, + achievement_evaluation="weekly") + + achievement_date1 = Achievement.get_datetime_for_evaluation_type(user1["timezone"], achievement["evaluation"]) + print("Achievement date for first user:") + print(achievement_date1) + create_achievement_user(user1, achievement, achievement_date1, level=1) + achievement_date2 = Achievement.get_datetime_for_evaluation_type(user2["timezone"], achievement["evaluation"]) + print("Achievement date for second user:") + print(achievement_date2) + create_achievement_user(user2, achievement, achievement_date2, level=1) + achievement_date3 = Achievement.get_datetime_for_evaluation_type(user3["timezone"], achievement["evaluation"]) + create_achievement_user(user3, achievement, achievement_date3, level=1) + achievement_date4 = Achievement.get_datetime_for_evaluation_type(user4["timezone"], achievement["evaluation"]) + create_achievement_user(user4, achievement, achievement_date4, level=1) + + create_variable("invite_users", variable_group="day") + Value.increase_value(variable_name="invite_users", user=user1, value=3, key=None) + Value.increase_value(variable_name="invite_users", user=user2, value=2, key=None) + Value.increase_value(variable_name="invite_users", user=user3, value=1, key=None) + Value.increase_value(variable_name="invite_users", user=user4, value=6, key=None) + + create_goals(achievement, + goal_goal="3*level", + goal_operator="geq", + goal_group_by_key=False + ) + + clear_all_caches() + + # Evaluate achievement for all users + achievement1 = Achievement.evaluate(user1, achievement.id, achievement_date1) + print(achievement1) + achievement2 = Achievement.evaluate(user2, achievement.id, achievement_date2) + print(achievement2) + achievement3 = Achievement.evaluate(user3, achievement.id, achievement_date3) + print(achievement3) + achievement4 = Achievement.evaluate(user4, achievement.id, achievement_date4) + print(achievement4) + + Value.increase_value(variable_name="invite_users", user=user1, value=6, key=None) + Value.increase_value(variable_name="invite_users", user=user2, value=5, key=None) + # Value.increase_value(variable_name="invite_users", user=user3, value=8, key=None) + # Value.increase_value(variable_name="invite_users", user=user4, value=2, key=None) + # + # print("In the next week for all users") + # achievement1_1 = Achievement.evaluate(user1, achievement.id, achievement_date1+datetime.timedelta(7)) + # print(achievement1_1) + # achievement2_1 = Achievement.evaluate(user2, achievement.id, achievement_date2+datetime.timedelta(7)) + # print(achievement2_1) + # achievement3_1 = Achievement.evaluate(user3, achievement.id, achievement_date3+datetime.timedelta(7)) + # print(achievement3_1) + # achievement4_1 = Achievement.evaluate(user4, achievement.id, achievement_date4+datetime.timedelta(7)) + # print(achievement4_1) \ No newline at end of file diff --git a/gengine/app/tests/test_goal.py b/gengine/app/tests/test_goal.py index 9551492..15a77c9 100644 --- a/gengine/app/tests/test_goal.py +++ b/gengine/app/tests/test_goal.py @@ -109,7 +109,7 @@ def test_get_goal_properties(self): self.assertNotEquals(result1, []) def test_get_leaderboard(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement") goals = create_goals(achievement) @@ -118,11 +118,11 @@ def test_get_leaderboard(self): user2 = create_user( lat=85.59, lng=65.75, - country="DE", - region="Niedersachsen", - city="Osnabrück", - timezone="Europe/Berlin", - language="de", + country="USA", + region="Lethal crosside", + city="New York", + timezone="US/Eastern", + language="en", additional_public_data={ "first_name": "Michael", "last_name": "Clarke" From de39387b10927625354835df9e27c35f2128e159 Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Thu, 16 Feb 2017 17:51:15 +0100 Subject: [PATCH 128/176] Adding changes in model for setting achievement_date while computing progress, Fixing timezone in achievemnt for leaderbord --- gengine/app/model.py | 67 +++++++++++-------- gengine/app/tests/helpers.py | 5 +- .../test_achievement_integration_tests.py | 64 +++++++----------- .../app/tests/test_eval_types_and_rewards.py | 48 +++++-------- gengine/app/tests/test_goal.py | 2 +- gengine/app/views.py | 4 +- 6 files changed, 87 insertions(+), 103 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 0759408..95482db 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -124,7 +124,8 @@ def get_default_token_valid_time(): Column("lng", ty.Float(Precision=64), nullable=True), Column("max_distance", ty.Integer, nullable=True), Column('priority', ty.Integer, index=True, default=0), - Column('evaluation',ty.Enum("immediately","daily","weekly","monthly","yearly","end", name="evaluation_types"), default="immediately", nullable=False), + Column('evaluation', ty.Enum("immediately","daily","weekly","monthly","yearly","end", name="evaluation_types"), default="immediately", nullable=False), + Column('evaluation_timezone', ty.String(), default=None, nullable=True), Column('relevance',ty.Enum("global","friends","city","own", name="relevance_types"), default="own"), Column('view_permission',ty.Enum("everyone", "own", name="achievement_view_permission"), default="everyone"), Column('created_at', ty.DateTime, nullable = False, default=datetime.datetime.utcnow), @@ -686,7 +687,7 @@ def invalidate_caches_for_variable_and_user(cls,variable_id,user_id): """ invalidate the relevant caches for this user and all relevant users with concerned leaderboards""" goalsandachievements = cls.map_variables_to_rules().get(variable_id,[]) - timezone = User.get_user(user_id)["timezone"] + timezone = "UTC" Goal.clear_goal_caches(user_id, [(entry["goal"]["id"],Achievement.get_datetime_for_evaluation_type(timezone, entry["achievement"]["evaluation"])) for entry in goalsandachievements]) for entry in goalsandachievements: achievement_date = Achievement.get_datetime_for_evaluation_type(timezone, entry["achievement"]["evaluation"]) @@ -725,15 +726,11 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): variable = Variable.get_variable_by_name(variable_name) dt = Variable.get_datetime_for_tz_and_group(tz,variable["group"],at_datetime=at_datetime) - print("dt in increase value") - print(dt) + condition = and_(t_values.c.datetime==dt, t_values.c.variable_id==variable["id"], t_values.c.user_id==user_id, t_values.c.key==str(key)) - datetime_db = DBSession.execute(select([t_values.c.datetime, ])).scalar() - print("datetime in db") - print(datetime_db) current_value = DBSession.execute(select([t_values.c.value,]).where(condition)).scalar() if current_value is not None: @@ -747,7 +744,6 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): Variable.invalidate_caches_for_variable_and_user(variable["id"],user["id"]) new_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() - print(new_value) return new_value class AchievementCategory(ABase): @@ -943,12 +939,10 @@ def generate(): all_goals_achieved = True goals = Goal.get_goals(achievement["id"]) for goal in goals: - print("goal_in_goals") goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) if not goal_eval: Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level,None) goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) - print(goal_eval) if achievement["relevance"]=="friends" or achievement["relevance"]=="city" or achievement["relevance"]=="global": goal_eval["leaderboard"] = Goal.get_leaderboard(goal, achievement_date, user_ids) @@ -1113,16 +1107,20 @@ def get_achievement_properties(cls,achievement_id,level): .fetchall() @classmethod - def get_datetime_for_evaluation_type(cls, tz, evaluation_type, dt=None): + def get_datetime_for_evaluation_type(cls, evaluation_timezone, evaluation_type, dt=None): """ This computes the datetime to identify the time of the achievement. Only relevant for repeating achievements (monthly, yearly, weekly, daily) Returns None for all other achievement types """ - tzobj = pytz.timezone(tz) + if evaluation_type and not evaluation_timezone: + evaluation_timezone = "UTC" + + tzobj = pytz.timezone(evaluation_timezone) if not dt: dt = datetime.datetime.now(tzobj) + else: dt = dt.astimezone(tzobj) @@ -1215,7 +1213,7 @@ def get_goals(cls,achievement_id): return DBSession.execute(t_goals.select(t_goals.c.achievement_id==achievement_id)).fetchall() @classmethod - def compute_progress(cls, goal, achievement, user_id): + def compute_progress(cls, goal, achievement, user, evaluation_date): """computes the progress of the goal for the given user_id goal attributes: @@ -1232,6 +1230,9 @@ def compute_progress(cls, goal, achievement, user_id): """ + user_id = user["id"] + timezone = "UTC" + def generate_statement_cache(): condition = evaluate_condition(goal["condition"], column_variable = t_variables.c.name.label("variable_name"), column_key = t_values.c.key) @@ -1253,6 +1254,7 @@ def generate_statement_cache(): datetime_col=None if group_by_dateformat: # here we need to convert to users' time zone, as we might need to group by e.g. USER's weekday + #TODO: modify when implementing fixed timezone for achievements datetime_col = func.to_char(text("values.datetime AT TIME ZONE users.timezone"),group_by_dateformat).label("datetime") select_cols.append(datetime_col) @@ -1274,14 +1276,26 @@ def generate_statement_cache(): if evaluation_type!="immediately": + achievement_date = Achievement.get_datetime_for_evaluation_type(timezone, evaluation_type, evaluation_date) if evaluation_type=="daily": q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("day","users.timezone"))) elif evaluation_type=="weekly": - q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("week","users.timezone"))) + q = q.where(and_( + t_values.c.datetime >= achievement_date, + t_values.c.datetime < achievement_date + datetime.timedelta(days=7)) + ) elif evaluation_type=="monthly": - q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("month","users.timezone"))) + next_month = Achievement.get_datetime_for_evaluation_type(timezone, "monthly", achievement_date + datetime.timedelta(days=32)) + q = q.where(and_( + t_values.c.datetime >= achievement_date, + t_values.c.datetime < next_month) + ) elif evaluation_type=="yearly": - q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("year","users.timezone"))) + next_year = Achievement.get_datetime_for_evaluation_type(timezone, "monthly", achievement_date + datetime.timedelta(days=366)) + q = q.where(and_( + t_values.c.datetime >= achievement_date, + t_values.c.datetime < next_year) + ) elif evaluation_type == "end": pass #Todo implement for end @@ -1311,16 +1325,20 @@ def generate_statement_cache(): #q = cache_goal_statements.get_or_create(str(goal["id"]),generate_statement_cache) # TODO: Cache the statement / Make it serializable for caching in redis q = generate_statement_cache() + return DBSession.execute(q, {'user_id' : user_id}) @classmethod def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval_cache_before=False, execute_triggers=True): """evaluate the goal for the user_ids and the level""" + operator = goal["operator"] - users_progress = Goal.compute_progress(goal, achievement, user_id) + #TODO: Move this call to outer loops + user = User.get_user(user_id) + + users_progress = Goal.compute_progress(goal, achievement, user, achievement_date) goal_evaluation = {e["user_id"] : e["value"] for e in users_progress} - print("user_progress ", goal_evaluation) goal_achieved = False if goal_eval_cache_before is False: @@ -1334,13 +1352,9 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval params = { "level" : level } - print("params", params) goal_goal = evaluate_value_expression(goal["goal"], params) - print("goal_goal",goal_goal) - print("user_value ", new) if goal_goal is not None and operator=="geq" and new>=goal_goal: goal_achieved = True - print("goal_achieved is ",goal_achieved) new = min(new,goal_goal) elif goal_goal is not None and operator=="leq" and new<=goal_goal: @@ -1348,7 +1362,6 @@ def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval new = max(new,goal_goal) previous_goal = Goal.basic_goal_output(goal, level-1).get("goal_goal",0) - print("previous_goal") # Evaluate triggers if execute_triggers: Goal.select_and_execute_triggers(goal = goal, user_id = user_id, level = level, current_goal = goal_goal, previous_goal = previous_goal, value = new) @@ -1439,7 +1452,6 @@ def get_goal_eval_cache(cls,goal_id,achievement_date,user_id): @classmethod def set_goal_eval_cache(cls,goal, achievement_date, user_id,value,achieved): """set cache entry after evaluation""" - cache_query = t_goal_evaluation_cache.select().where(and_(t_goal_evaluation_cache.c.goal_id==goal["id"], t_goal_evaluation_cache.c.user_id==user_id, t_goal_evaluation_cache.c.achievement_date==achievement_date)) @@ -1505,11 +1517,12 @@ def clear_goal_caches(cls, user_id, goal_ids_with_achievement_date): @classmethod def get_leaderboard(cls, goal, achievement_date, user_ids): """get the leaderboard for the goal and userids""" + q = select([t_goal_evaluation_cache.c.user_id, t_goal_evaluation_cache.c.value])\ .where(and_(t_goal_evaluation_cache.c.user_id.in_(user_ids), t_goal_evaluation_cache.c.goal_id==goal["id"], - t_goal_evaluation_cache.c.achievement_date==achievement_date))\ + ))\ .order_by(t_goal_evaluation_cache.c.value.desc(), t_goal_evaluation_cache.c.user_id.desc()) items = DBSession.execute(q).fetchall() @@ -1525,7 +1538,7 @@ def get_leaderboard(cls, goal, achievement_date, user_ids): user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) user_wants_level = min((user_has_level or 0)+1, achievement["maxlevel"]) - goal_eval = Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level) + Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level) #rerun the query items = DBSession.execute(q).fetchall() @@ -1706,7 +1719,7 @@ def insert_trigger_step_executions_after_step_upsert(mapper,connection,target): achievement = goal.achievement for user_id in user_ids: - achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user_id)["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(evaluation_timezone="UTC", evaluation_type=achievement["evaluation"]) user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) user_wants_level = min((user_has_level or 0) + 1, achievement["maxlevel"]) goal_eval = Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level, None, execute_triggers=False) diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index e88cfbf..0366cd4 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -236,7 +236,7 @@ def create_achievement( achievement_evaluation = undefined, achievement_relevance = undefined, achievement_maxlevel = undefined, - achievement_view_permission = undefined, + achievement_view_permission = undefined ): achievement = Achievement() @@ -290,6 +290,8 @@ def create_achievement( else: achievement.view_permission = achievement_view_permission + achievement.evaluation_timezone = "UTC" + DBSession.add(achievement) DBSession.flush() @@ -425,6 +427,7 @@ def create_achievement_user(user, achievement, achievement_date, level): return achievement_user + def create_goal_evaluation_cache( goal_id , gec_achievement_date, diff --git a/gengine/app/tests/test_achievement_integration_tests.py b/gengine/app/tests/test_achievement_integration_tests.py index de290cd..3ba2b2a 100644 --- a/gengine/app/tests/test_achievement_integration_tests.py +++ b/gengine/app/tests/test_achievement_integration_tests.py @@ -12,7 +12,7 @@ class TestAchievementEvaluationType(BaseDBTest): # Case1: Achieved in first and next week def test_evaluate_achievement_for_weekly_evaluation_case1(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -42,17 +42,16 @@ def test_evaluate_achievement_for_weekly_evaluation_case1(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print(achievement_result) - update_connection().execute(t_values.delete()) - DBSession.flush() - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="weekly", dt=next_weekdate) - Value.increase_value(variable_name="invite_users", user=user, value=16, key=None) + + Value.increase_value(variable_name="invite_users", user=user, value=16, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print(achievement_result1) + # Case2: NOT Achieved in first week but in next week def test_evaluate_achievement_for_weekly_evaluation_case2(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -80,17 +79,14 @@ def test_evaluate_achievement_for_weekly_evaluation_case2(self): achievement_result2 = Achievement.evaluate(user, achievement.id, achievement_date) print(achievement_result2) - update_connection().execute(t_values.delete()) - DBSession.flush() - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="weekly", dt=next_weekdate+datetime.timedelta(3)) - Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) achievement_resul1 = Achievement.evaluate(user, achievement.id, next_date) print("achievement result1: ", achievement_resul1) # Case3: NOT Achieved in first week but after some days in same week def test_evaluate_achievement_for_weekly_evaluation_case3(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -118,13 +114,13 @@ def test_evaluate_achievement_for_weekly_evaluation_case3(self): print(achievement_result) next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="weekly", dt=achievement_date+datetime.timedelta(3)) - Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) achievement_resul1 = Achievement.evaluate(user, achievement.id, next_date) print("achievement result1: ", achievement_resul1) # Case1: Achieved in first and next month def test_evaluate_achievement_for_monthly_evaluation_case1(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -155,17 +151,15 @@ def test_evaluate_achievement_for_monthly_evaluation_case1(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - update_connection().execute(t_values.delete()) - DBSession.flush() - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="monthly", dt=next_month) - Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) achievement_resul1 = Achievement.evaluate(user, achievement.id, next_date) print("achievement result1: ", achievement_resul1) # Case2: Not achieved in first but in next month def test_evaluate_achievement_for_monthly_evaluation_case2(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -196,17 +190,15 @@ def test_evaluate_achievement_for_monthly_evaluation_case2(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - update_connection().execute(t_values.delete()) - DBSession.flush() - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="monthly", dt=next_month+datetime.timedelta(10)) - Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print("achievement result1: ", achievement_result1) # Case2: Achieved in first month and after some days in a same month def test_evaluate_achievement_for_monthly_evaluation_case3(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -236,13 +228,13 @@ def test_evaluate_achievement_for_monthly_evaluation_case3(self): print("achievement result: ", achievement_result) next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="monthly", dt=achievement_date+datetime.timedelta(10)) - Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print("achievement result1: ", achievement_result1) # Case1: Achieved in first year and next year def test_evaluate_achievement_for_yearly_evaluation_case1(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -273,17 +265,15 @@ def test_evaluate_achievement_for_yearly_evaluation_case1(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - update_connection().execute(t_values.delete()) - DBSession.flush() - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="yearly", dt=next_year) - Value.increase_value(variable_name="invite_users", user=user, value=15, key=None) + + Value.increase_value(variable_name="invite_users", user=user, value=15, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print(achievement_result1) # Case2: Not Achieved in first year but in next year def test_evaluate_achievement_for_yearly_evaluation_case2(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -314,17 +304,15 @@ def test_evaluate_achievement_for_yearly_evaluation_case2(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - update_connection().execute(t_values.delete()) - DBSession.flush() - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="yearly", dt=next_year + datetime.timedelta(10)) - Value.increase_value(variable_name="invite_users", user=user, value=15, key=None) + + Value.increase_value(variable_name="invite_users", user=user, value=15, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print("achievement result1: ", achievement_result1) # Case3: Achieved in this year and after some days in same year def test_evaluate_achievement_for_yearly_evaluation_case3(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -355,10 +343,8 @@ def test_evaluate_achievement_for_yearly_evaluation_case3(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - update_connection().execute(t_values.delete()) - DBSession.flush() + next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="yearly", dt=achievement_date + datetime.timedelta(110)) - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="yearly", dt=achievement_date + datetime.timedelta(10)) - Value.increase_value(variable_name="invite_users", user=user, value=10, key=None) + Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print("achievement result1: ", achievement_result1) \ No newline at end of file diff --git a/gengine/app/tests/test_eval_types_and_rewards.py b/gengine/app/tests/test_eval_types_and_rewards.py index cb0574b..6d51849 100644 --- a/gengine/app/tests/test_eval_types_and_rewards.py +++ b/gengine/app/tests/test_eval_types_and_rewards.py @@ -3,7 +3,7 @@ from gengine.app.cache import clear_all_caches from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals, create_achievement_rewards, create_achievement_user +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals, create_achievement_rewards, create_achievement_user, create_goal_evaluation_cache from gengine.metadata import DBSession from gengine.app.model import Achievement, AchievementUser, Value, t_values from gengine.base.model import update_connection @@ -11,7 +11,7 @@ class TestAchievementEvaluationTypeAndRewards(BaseDBTest): def test_multiple_users_achievemnt_reward(self): - return + user1 = create_user() # Create Second user @@ -66,23 +66,20 @@ def test_multiple_users_achievemnt_reward(self): achievement_maxlevel=3, achievement_evaluation="weekly") - achievement_date1 = Achievement.get_datetime_for_evaluation_type(user1["timezone"], achievement["evaluation"]) + print(achievement.evaluation_timezone) + achievement_date1 = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation) print("Achievement date for first user:") print(achievement_date1) + create_achievement_user(user1, achievement, achievement_date1, level=1) - achievement_date2 = Achievement.get_datetime_for_evaluation_type(user2["timezone"], achievement["evaluation"]) - print("Achievement date for second user:") - print(achievement_date2) - create_achievement_user(user2, achievement, achievement_date2, level=1) - achievement_date3 = Achievement.get_datetime_for_evaluation_type(user3["timezone"], achievement["evaluation"]) - create_achievement_user(user3, achievement, achievement_date3, level=1) - achievement_date4 = Achievement.get_datetime_for_evaluation_type(user4["timezone"], achievement["evaluation"]) - create_achievement_user(user4, achievement, achievement_date4, level=1) + create_achievement_user(user2, achievement, achievement_date1, level=1) + create_achievement_user(user3, achievement, achievement_date1, level=1) + create_achievement_user(user4, achievement, achievement_date1, level=1) create_variable("invite_users", variable_group="day") Value.increase_value(variable_name="invite_users", user=user1, value=3, key=None) Value.increase_value(variable_name="invite_users", user=user2, value=2, key=None) - Value.increase_value(variable_name="invite_users", user=user3, value=1, key=None) + Value.increase_value(variable_name="invite_users", user=user3, value=11, key=None) Value.increase_value(variable_name="invite_users", user=user4, value=6, key=None) create_goals(achievement, @@ -94,26 +91,11 @@ def test_multiple_users_achievemnt_reward(self): clear_all_caches() # Evaluate achievement for all users - achievement1 = Achievement.evaluate(user1, achievement.id, achievement_date1) + achievement1 = Achievement.evaluate(user3, achievement.id, achievement_date1) print(achievement1) - achievement2 = Achievement.evaluate(user2, achievement.id, achievement_date2) - print(achievement2) - achievement3 = Achievement.evaluate(user3, achievement.id, achievement_date3) - print(achievement3) - achievement4 = Achievement.evaluate(user4, achievement.id, achievement_date4) - print(achievement4) - Value.increase_value(variable_name="invite_users", user=user1, value=6, key=None) - Value.increase_value(variable_name="invite_users", user=user2, value=5, key=None) - # Value.increase_value(variable_name="invite_users", user=user3, value=8, key=None) - # Value.increase_value(variable_name="invite_users", user=user4, value=2, key=None) - # - # print("In the next week for all users") - # achievement1_1 = Achievement.evaluate(user1, achievement.id, achievement_date1+datetime.timedelta(7)) - # print(achievement1_1) - # achievement2_1 = Achievement.evaluate(user2, achievement.id, achievement_date2+datetime.timedelta(7)) - # print(achievement2_1) - # achievement3_1 = Achievement.evaluate(user3, achievement.id, achievement_date3+datetime.timedelta(7)) - # print(achievement3_1) - # achievement4_1 = Achievement.evaluate(user4, achievement.id, achievement_date4+datetime.timedelta(7)) - # print(achievement4_1) \ No newline at end of file + new_date = achievement_date1+datetime.timedelta(7) + print(new_date) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation, dt=new_date) + achievement = Achievement.evaluate(user3, achievement.id, next_date) + print(achievement) diff --git a/gengine/app/tests/test_goal.py b/gengine/app/tests/test_goal.py index 15a77c9..fa59686 100644 --- a/gengine/app/tests/test_goal.py +++ b/gengine/app/tests/test_goal.py @@ -109,7 +109,7 @@ def test_get_goal_properties(self): self.assertNotEquals(result1, []) def test_get_leaderboard(self): - + return achievement = create_achievement(achievement_name="invite_users_achievement") goals = create_goals(achievement) diff --git a/gengine/app/views.py b/gengine/app/views.py index 085309a..a7d4d27 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -163,7 +163,7 @@ def may_view(achievement, requesting_user): achievement_dates = set() d = max(achievement["created_at"],achievements_for_user["created_at"]).replace(tzinfo=pytz.utc) dr = Achievement.get_datetime_for_evaluation_type( - achievements_for_user["timezone"], + "UTC", achievement["evaluation"], dt=d ) @@ -185,7 +185,7 @@ def may_view(achievement, requesting_user): break # should not happen dr = Achievement.get_datetime_for_evaluation_type( - achievements_for_user["timezone"], + "UTC", achievement["evaluation"], dt=d ) From 4d7450b7f00fc39790ca814b88ba4ce3f2e3987a Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Fri, 17 Feb 2017 14:16:28 +0100 Subject: [PATCH 129/176] fix write session --- gengine/base/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/base/model.py b/gengine/base/model.py index 146a9ef..0ef8f4d 100644 --- a/gengine/base/model.py +++ b/gengine/base/model.py @@ -112,6 +112,6 @@ def valid_timezone(timezone): def update_connection(): - session = meta.DBSession + session = meta.DBSession() mark_changed(session) return session From 90950d75148c6689f88941870ef97927658c9285 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Fri, 17 Feb 2017 14:16:43 +0100 Subject: [PATCH 130/176] database migration for evaluation_timezones --- .../62026366cd60_evaluation_timezone.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 gengine/app/alembic/versions/62026366cd60_evaluation_timezone.py diff --git a/gengine/app/alembic/versions/62026366cd60_evaluation_timezone.py b/gengine/app/alembic/versions/62026366cd60_evaluation_timezone.py new file mode 100644 index 0000000..00d50b4 --- /dev/null +++ b/gengine/app/alembic/versions/62026366cd60_evaluation_timezone.py @@ -0,0 +1,28 @@ +"""evaluation_timezone + +Revision ID: 62026366cd60 +Revises: 87dfedb58883 +Create Date: 2017-02-17 13:54:40.545893 + +""" + +# revision identifiers, used by Alembic. +revision = '62026366cd60' +down_revision = '87dfedb58883' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('achievements', sa.Column('evaluation_timezone', sa.String(), nullable=True)) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('achievements', 'evaluation_timezone') + ### end Alembic commands ### From 7fe501a5f61c6634ef4c9c3c65e64cc217d3faab Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Fri, 17 Feb 2017 14:16:58 +0100 Subject: [PATCH 131/176] work on passing the evaluation_timezone --- gengine/app/model.py | 20 +++++++++++++------- gengine/app/views.py | 4 ++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 94f731d..be398d6 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -688,10 +688,13 @@ def invalidate_caches_for_variable_and_user(cls,variable_id,user_id): """ invalidate the relevant caches for this user and all relevant users with concerned leaderboards""" goalsandachievements = cls.map_variables_to_rules().get(variable_id,[]) - timezone = "UTC" - Goal.clear_goal_caches(user_id, [(entry["goal"]["id"],Achievement.get_datetime_for_evaluation_type(timezone, entry["achievement"]["evaluation"])) for entry in goalsandachievements]) + Goal.clear_goal_caches(user_id, [ + (entry["goal"]["id"], Achievement.get_datetime_for_evaluation_type(entry["achievement"]["evaluation_timezone"], entry["achievement"]["evaluation"])) + for entry in goalsandachievements + ] + ) for entry in goalsandachievements: - achievement_date = Achievement.get_datetime_for_evaluation_type(timezone, entry["achievement"]["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(entry["achievement"]["evaluation_timezone"], entry["achievement"]["evaluation"]) Achievement.invalidate_evaluate_cache(user_id, entry["achievement"], achievement_date) @classmethod @@ -1232,7 +1235,7 @@ def compute_progress(cls, goal, achievement, user, evaluation_date): """ user_id = user["id"] - timezone = "UTC" + timezone = achievement["evaluation_timezone"] def generate_statement_cache(): condition = evaluate_condition(goal["condition"], column_variable = t_variables.c.name.label("variable_name"), @@ -1279,7 +1282,10 @@ def generate_statement_cache(): achievement_date = Achievement.get_datetime_for_evaluation_type(timezone, evaluation_type, evaluation_date) if evaluation_type=="daily": - q = q.where(text("values.datetime AT TIME ZONE users.timezone>"+datetime_trunc("day","users.timezone"))) + q = q.where(and_( + t_values.c.datetime >= achievement_date, + t_values.c.datetime < achievement_date + datetime.timedelta(days=1)) + ) elif evaluation_type=="weekly": q = q.where(and_( t_values.c.datetime >= achievement_date, @@ -1292,7 +1298,7 @@ def generate_statement_cache(): t_values.c.datetime < next_month) ) elif evaluation_type=="yearly": - next_year = Achievement.get_datetime_for_evaluation_type(timezone, "monthly", achievement_date + datetime.timedelta(days=366)) + next_year = Achievement.get_datetime_for_evaluation_type(timezone, "yearly", achievement_date + datetime.timedelta(days=366)) q = q.where(and_( t_values.c.datetime >= achievement_date, t_values.c.datetime < next_year) @@ -1731,7 +1737,7 @@ def insert_trigger_step_executions_after_step_upsert(mapper,connection,target): achievement = goal.achievement for user_id in user_ids: - achievement_date = Achievement.get_datetime_for_evaluation_type(evaluation_timezone="UTC", evaluation_type=achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(evaluation_timezone=achievement["evaluation_timezone"], evaluation_type=achievement["evaluation"]) user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) user_wants_level = min((user_has_level or 0) + 1, achievement["maxlevel"]) goal_eval = Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level, None, execute_triggers=False) diff --git a/gengine/app/views.py b/gengine/app/views.py index 6d2cf55..1a9b9d4 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -162,7 +162,7 @@ def may_view(achievement, requesting_user): achievement_dates = set() d = max(achievement["created_at"], achievements_for_user["created_at"]).replace(tzinfo=pytz.utc) dr = Achievement.get_datetime_for_evaluation_type( - "UTC", + achievement["evaluation_timezone"], achievement["evaluation"], dt=d ) @@ -184,7 +184,7 @@ def may_view(achievement, requesting_user): break # should not happen dr = Achievement.get_datetime_for_evaluation_type( - "UTC", + achievement["evaluation_timezone"], achievement["evaluation"], dt=d ) From f3d90d71d90539b21ad1739d3950317ef57cd928 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Fri, 17 Feb 2017 14:50:48 +0100 Subject: [PATCH 132/176] tests: init db with scoped_session --- gengine/app/tests/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gengine/app/tests/base.py b/gengine/app/tests/base.py index 949065f..c58470b 100644 --- a/gengine/app/tests/base.py +++ b/gengine/app/tests/base.py @@ -2,7 +2,8 @@ import os from sqlalchemy.engine import create_engine from sqlalchemy.sql.schema import Table -from gengine.metadata import init_db, init_session, get_sessionmaker +from sqlalchemy.orm.scoping import scoped_session +from gengine.metadata import init_session, get_sessionmaker from gengine.app.tests import db class BaseDBTest(unittest.TestCase): @@ -24,7 +25,7 @@ def setUp(self): "database": dsn["database"], } ) - init_session(override_session=get_sessionmaker()(bind=self.engine), replace=True) + init_session(override_session=scoped_session(get_sessionmaker()(bind=self.engine)), replace=True) from gengine.metadata import Base Base.metadata.bind = self.engine From 814eae3c72a7588d9578ff42be17914bb5fe8016 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Fri, 17 Feb 2017 14:53:08 +0100 Subject: [PATCH 133/176] pass bind parameter to sessionmaker --- gengine/app/tests/base.py | 2 +- gengine/metadata.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gengine/app/tests/base.py b/gengine/app/tests/base.py index c58470b..dec3e6b 100644 --- a/gengine/app/tests/base.py +++ b/gengine/app/tests/base.py @@ -25,7 +25,7 @@ def setUp(self): "database": dsn["database"], } ) - init_session(override_session=scoped_session(get_sessionmaker()(bind=self.engine)), replace=True) + init_session(override_session=scoped_session(get_sessionmaker(bind=self.engine)), replace=True) from gengine.metadata import Base Base.metadata.bind = self.engine diff --git a/gengine/metadata.py b/gengine/metadata.py index e2ba543..d0f0c87 100644 --- a/gengine/metadata.py +++ b/gengine/metadata.py @@ -18,10 +18,11 @@ def rollback(self,*args,**kw): DBSession=Proxy() -def get_sessionmaker(): +def get_sessionmaker(bind=None): return sessionmaker( extension=ZopeTransactionExtension(), - class_=MySession + class_=MySession, + bind=bind ) def init_session(override_session=None, replace=False): From 3463e3a900492155c98e14be63030bd76337fac4 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Fri, 17 Feb 2017 14:56:36 +0100 Subject: [PATCH 134/176] performance improvements --- gengine/app/model.py | 26 ++++++++++++++------------ gengine/app/tests/test_goal.py | 6 +++--- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index be398d6..0bfc478 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -945,7 +945,7 @@ def generate(): for goal in goals: goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) if not goal_eval: - Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level,None, execute_triggers=execute_triggers) + Goal.evaluate(goal, achievement, achievement_date, user, user_wants_level,None, execute_triggers=execute_triggers) goal_eval = Goal.get_goal_eval_cache(goal["id"], achievement_date, user_id) if achievement["relevance"]=="friends" or achievement["relevance"]=="city" or achievement["relevance"]=="global": @@ -1259,7 +1259,7 @@ def generate_statement_cache(): if group_by_dateformat: # here we need to convert to users' time zone, as we might need to group by e.g. USER's weekday #TODO: modify when implementing fixed timezone for achievements - datetime_col = func.to_char(text("values.datetime AT TIME ZONE users.timezone"),group_by_dateformat).label("datetime") + datetime_col = func.to_char(text("values.datetime AT TIME ZONE '%s'" % (timezone,)), group_by_dateformat).label("datetime") select_cols.append(datetime_col) if group_by_key: @@ -1336,13 +1336,13 @@ def generate_statement_cache(): return DBSession.execute(q, {'user_id' : user_id}) @classmethod - def evaluate(cls, goal, achievement, achievement_date, user_id, level, goal_eval_cache_before=False, execute_triggers=True): + def evaluate(cls, goal, achievement, achievement_date, user, level, goal_eval_cache_before=False, execute_triggers=True): """evaluate the goal for the user_ids and the level""" operator = goal["operator"] #TODO: Move this call to outer loops - user = User.get_user(user_id) + user_id = user["id"] users_progress = Goal.compute_progress(goal, achievement, user, achievement_date) goal_evaluation = {e["user_id"] : e["value"] for e in users_progress} @@ -1546,16 +1546,17 @@ def get_leaderboard(cls, goal, achievement_date, user_ids): users = User.get_users(user_ids) - missing_users = set(user_ids)-set([x["user_id"] for x in items]) + missing_user_ids = set(user_ids)-set([x["user_id"] for x in items]) + missing_users = User.get_users(missing_user_ids).values() if len(missing_users)>0: #the goal has not been evaluated for some users... achievement = Achievement.get_achievement(goal["achievement_id"]) - for user_id in missing_users: - user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) + for user in missing_users: + user_has_level = Achievement.get_level_int(user["id"], achievement["id"], achievement_date) user_wants_level = min((user_has_level or 0)+1, achievement["maxlevel"]) - Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level) + Goal.evaluate(goal, achievement, achievement_date, user, user_wants_level) #rerun the query items = DBSession.execute(q).fetchall() @@ -1733,14 +1734,15 @@ def insert_trigger_step_executions_after_step_upsert(mapper,connection,target): """When we create a new Trigger-Step, we must ensure, that is will not be executed for the users who already met the conditions before.""" user_ids = [x["id"] for x in DBSession.execute(select([t_users.c.id,],from_obj=t_users)).fetchall()] + users = User.get_users(user_ids).values() goal = target.trigger.goal achievement = goal.achievement - for user_id in user_ids: + for user in users: achievement_date = Achievement.get_datetime_for_evaluation_type(evaluation_timezone=achievement["evaluation_timezone"], evaluation_type=achievement["evaluation"]) - user_has_level = Achievement.get_level_int(user_id, achievement["id"], achievement_date) + user_has_level = Achievement.get_level_int(user["id"], achievement["id"], achievement_date) user_wants_level = min((user_has_level or 0) + 1, achievement["maxlevel"]) - goal_eval = Goal.evaluate(goal, achievement, achievement_date, user_id, user_wants_level, None, execute_triggers=False) + goal_eval = Goal.evaluate(goal, achievement, achievement_date, user, user_wants_level, None, execute_triggers=False) previous_goal = Goal.basic_goal_output(goal, user_wants_level - 1).get("goal_goal", 0) if previous_goal == goal_eval["goal_goal"]: @@ -1754,7 +1756,7 @@ def insert_trigger_step_executions_after_step_upsert(mapper,connection,target): or (operator == "leq" and current_percentage <= required_percentage): GoalTriggerStep.execute( trigger_step=target, - user_id=user_id, + user_id=user["id"], current_percentage=current_percentage, value=goal_eval["value"], goal_goal=goal_eval["goal_goal"], diff --git a/gengine/app/tests/test_goal.py b/gengine/app/tests/test_goal.py index fa59686..1e1ca5d 100644 --- a/gengine/app/tests/test_goal.py +++ b/gengine/app/tests/test_goal.py @@ -62,7 +62,7 @@ def test_evaluate_goal(self): goal = create_goals(achievement, goal_group_by_key=False, goal_goal="3*level") achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) - evaluation_result = Goal.evaluate(goal, achievement, achievement_date, user.id, level=4, goal_eval_cache_before=False, execute_triggers=True) + evaluation_result = Goal.evaluate(goal, achievement, achievement_date, user, level=4, goal_eval_cache_before=False, execute_triggers=True) print(evaluation_result) # True cases self.assertGreaterEqual(evaluation_result["value"], 12) @@ -70,7 +70,7 @@ def test_evaluate_goal(self): # Goal Participate with group_by = True goal2 = create_goals(achievement, goal_group_by_key=True, goal_goal="3*level") - evaluation_result2 = Goal.evaluate(goal2, achievement, achievement_date, user.id, level=4, goal_eval_cache_before=False, execute_triggers=True) + evaluation_result2 = Goal.evaluate(goal2, achievement, achievement_date, user, level=4, goal_eval_cache_before=False, execute_triggers=True) print(evaluation_result2) # failing cases self.assertGreaterEqual(evaluation_result2["value"], 12) @@ -81,7 +81,7 @@ def test_evaluate_goal(self): goal1 = create_goals(achievement1, goal_goal="4*level") achievement_date1 = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement1["evaluation"]) - evaluation_result1 = Goal.evaluate(goal1, achievement1, achievement_date1, user.id, level=2, goal_eval_cache_before=False, execute_triggers=True) + evaluation_result1 = Goal.evaluate(goal1, achievement1, achievement_date1, user, level=2, goal_eval_cache_before=False, execute_triggers=True) print(evaluation_result1) # True cases From a36fc8ec090543174796c82a5cc1c2d755183e0d Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Fri, 17 Feb 2017 15:01:28 +0100 Subject: [PATCH 135/176] fix group by dateformat for evaluation_timezone --- gengine/app/model.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 0bfc478..87cc9b7 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1259,7 +1259,11 @@ def generate_statement_cache(): if group_by_dateformat: # here we need to convert to users' time zone, as we might need to group by e.g. USER's weekday #TODO: modify when implementing fixed timezone for achievements - datetime_col = func.to_char(text("values.datetime AT TIME ZONE '%s'" % (timezone,)), group_by_dateformat).label("datetime") + if timezone: + datetime_col = func.to_char(text("values.datetime AT TIME ZONE '%s'" % (timezone,)), group_by_dateformat).label("datetime") + else: + datetime_col = func.to_char(text("values.datetime AT TIME ZONE users.timezone"), + group_by_dateformat).label("datetime") select_cols.append(datetime_col) if group_by_key: From 06e0f2b26e9786dc4023eabc762899b3c3150892 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Fri, 17 Feb 2017 15:02:45 +0100 Subject: [PATCH 136/176] remove todo note --- gengine/app/model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 87cc9b7..72c5610 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1258,7 +1258,6 @@ def generate_statement_cache(): datetime_col=None if group_by_dateformat: # here we need to convert to users' time zone, as we might need to group by e.g. USER's weekday - #TODO: modify when implementing fixed timezone for achievements if timezone: datetime_col = func.to_char(text("values.datetime AT TIME ZONE '%s'" % (timezone,)), group_by_dateformat).label("datetime") else: From 51a9c1cc030215892d31740ffdee1f016cbda28d Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Fri, 24 Feb 2017 01:03:24 +0100 Subject: [PATCH 137/176] refactoring code, adding correct assert statements, fixing issues in old test cases --- gengine/app/tests/helpers.py | 6 +- gengine/app/tests/test_achievement1.py | 72 ++++---- .../test_achievement_integration_tests.py | 171 +++++++++++++----- gengine/app/tests/test_auth.py | 9 +- gengine/app/tests/test_device.py | 4 - .../app/tests/test_eval_types_and_rewards.py | 35 +++- gengine/app/tests/test_goal.py | 36 ++-- 7 files changed, 198 insertions(+), 135 deletions(-) diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index 0366cd4..fc06769 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -71,8 +71,8 @@ def create_user( ): if additional_public_data is undefined: additional_public_data = { - 'first_name' : names.get_first_name(), - 'last_name' : names.get_last_name() + 'first_name' : 'Stefan', + 'last_name' : 'Rogers' } if user_id is undefined: @@ -251,7 +251,7 @@ def create_achievement( achievement.valid_start = achievement_valid_start if achievement_valid_end is undefined: - achievement.valid_end = datetime.datetime.utcnow() + achievement.valid_end = datetime.date.today() else: achievement.valid_end = achievement_valid_end diff --git a/gengine/app/tests/test_achievement1.py b/gengine/app/tests/test_achievement1.py index 88ae9a2..ff9a8c4 100644 --- a/gengine/app/tests/test_achievement1.py +++ b/gengine/app/tests/test_achievement1.py @@ -3,7 +3,7 @@ from gengine.app.cache import clear_all_caches from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_goals, create_achievement_rewards +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_goals, create_achievement_rewards, create_achievement_user from gengine.metadata import DBSession from gengine.app.model import Achievement, User, AchievementUser, Value, AchievementReward, Reward, AchievementProperty, AchievementAchievementProperty, t_values from gengine.base.model import update_connection @@ -12,20 +12,21 @@ class TestAchievement(BaseDBTest): # Includes get_achievement_by_location and get_achievement_by_date def test_get_achievements_by_location_and_date(self): - return + user = create_user() achievement1 = create_achievement(achievement_name="invite_users_achievement") achievement2 = create_achievement(achievement_name="participate_achievement") - DBSession.flush() create_goals(achievement1) + create_goals(achievement2) achievement_today = Achievement.get_achievements_by_user_for_today(user) + print("achievement_today") print(achievement_today) self.assertEqual(achievement_today[0]["name"], "invite_users_achievement") self.assertEqual(len(achievement_today), 2) def test_get_relevant_users_by_achievement_friends_and_user(self): - return + #Create First user user1 = create_user() @@ -81,23 +82,23 @@ def test_get_relevant_users_by_achievement_friends_and_user(self): friendsOfuser3 = achievement.get_relevant_users_by_achievement_and_user(achievement, user3.id) friendsOfuser4 = achievement.get_relevant_users_by_achievement_and_user(achievement, user4.id) - self.assertNotIn(1, friendsOfuser1) - self.assertNotIn(2, friendsOfuser1) - self.assertNotIn(4, friendsOfuser3) - self.assertNotIn(2, friendsOfuser3) - self.assertNotIn(3, friendsOfuser4) - self.assertNotIn(2, friendsOfuser4) + self.assertIn(1, friendsOfuser1) + self.assertIn(1, friendsOfuser3) + self.assertIn(2, friendsOfuser3) + self.assertIn(3, friendsOfuser4) # For the relevance global achievement1 = create_achievement(achievement_relevance = "global") - DBSession.add(achievement1) friendsOfuser1 = achievement.get_relevant_users_by_achievement_and_user(achievement1, user3.id) - self.assertNotIn(1, friendsOfuser1) + self.assertIn(1, friendsOfuser1) + self.assertIn(2, friendsOfuser1) + self.assertIn(3, friendsOfuser1) + self.assertIn(4, friendsOfuser1) def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): - return + # Create First user user1 = create_user() @@ -114,7 +115,7 @@ def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): "first_name": "Michael", "last_name": "Clarke" }, - friends = [1] + friends = [1, 4] ) # Create Third user @@ -160,43 +161,34 @@ def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): print(usersForFriend3) print(usersForFriend4) - self.assertNotIn(4, usersForFriend1) - self.assertNotIn(2, usersForFriend1) - self.assertNotIn(1, usersForFriend2) - self.assertNotIn(3, usersForFriend2) - self.assertNotIn(1, usersForFriend3) - self.assertNotIn(4, usersForFriend3) - self.assertNotIn(2, usersForFriend4) - self.assertNotIn(4, usersForFriend4) + self.assertIn(2, usersForFriend1) + # self.assertIn(3, usersForFriend1) + self.assertIn(3, usersForFriend2) + self.assertIn(4, usersForFriend2) + self.assertIn(4, usersForFriend3) + self.assertIn(1, usersForFriend4) + self.assertIn(4, usersForFriend4) def test_get_level(self): - return + user = create_user(timezone="Australia/Sydney", country="Australia", region="xyz", city="Sydney") - achievement = create_achievement(achievement_name="invite_users_achievement") + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_evaluation="weekly") - achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation) - achievement_users = AchievementUser() - achievement_users.user_id = user.id - achievement_users.achievement_id = achievement["id"] - achievement_users.achievement_date = achievement_date - achievement_users.level = 2 - DBSession.add(achievement_users) - DBSession.flush() + create_achievement_user(user, achievement, achievement_date, level=2) achievement.get_level(user.id, achievement["id"], achievement_date) level = achievement.get_level_int(user.id, achievement.id, achievement_date) - print("level:",level) - achievement1 = create_achievement(achievement_evaluation="weekly") - achievement_date1 = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement1["evaluation"]) + achievement_date1 = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement_date+datetime.timedelta(7)) print("achievement date") print(achievement_date1) achievement.get_level(user.id, achievement["id"], achievement_date1) + level1 = achievement.get_level_int(user.id, achievement.id, achievement_date1) # Test for get_level as integer - level1 = achievement.get_level_int(user.id, achievement.id, achievement_date1) print("level1:", level1) self.assertEqual(level, 2) self.assertEqual(level1, 0) @@ -257,7 +249,7 @@ def test_evaluate_achievement_for_participate(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation) current_level = 1 achievement_user = AchievementUser() @@ -296,7 +288,7 @@ def test_evaluate_achievement_for_invite_users(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation) # get level current_level = 1 @@ -367,7 +359,7 @@ def test_get_reward_and_properties_for_achievement(self): DBSession.add(achievement_reward) DBSession.flush() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation) current_level = 1 achievement_user = AchievementUser() @@ -424,7 +416,7 @@ def test_multiple_goals_of_same_achievement(self): DBSession.add(achievement_reward) DBSession.flush() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation) goal1 = create_goals(achievement=achievement, goal_condition="""{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate_seminar"}}""", diff --git a/gengine/app/tests/test_achievement_integration_tests.py b/gengine/app/tests/test_achievement_integration_tests.py index 3ba2b2a..d3947f4 100644 --- a/gengine/app/tests/test_achievement_integration_tests.py +++ b/gengine/app/tests/test_achievement_integration_tests.py @@ -1,18 +1,15 @@ import datetime -import pytz - from gengine.app.cache import clear_all_caches from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals, create_achievement_rewards, create_achievement_user -from gengine.metadata import DBSession -from gengine.app.model import Achievement, AchievementUser, Value, t_values -from gengine.base.model import update_connection +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_goals, create_achievement_user +from gengine.app.model import Achievement, Value + class TestAchievementEvaluationType(BaseDBTest): # Case1: Achieved in first and next week def test_evaluate_achievement_for_weekly_evaluation_case1(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -20,9 +17,9 @@ def test_evaluate_achievement_for_weekly_evaluation_case1(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement["evaluation"]) print(achievement_date) - next_weekdate = achievement_date + datetime.timedelta(7) + next_weekdate = achievement_date + datetime.timedelta(10) print(next_weekdate) create_achievement_user(user, achievement, achievement_date, level=1) @@ -42,16 +39,25 @@ def test_evaluate_achievement_for_weekly_evaluation_case1(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print(achievement_result) - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="weekly", dt=next_weekdate) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="weekly", dt=next_weekdate) Value.increase_value(variable_name="invite_users", user=user, value=16, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print(achievement_result1) + self.assertEqual(achievement_result["achievement_date"], achievement_date) + self.assertEqual(achievement_result1["achievement_date"], next_date) + self.assertNotEqual(next_weekdate, next_date) + self.assertIn('1', achievement_result["levels_achieved"]) + self.assertIn('2', achievement_result["levels_achieved"]) + self.assertIn('3', achievement_result["levels_achieved"]) + self.assertIn('1', achievement_result1["new_levels"]) + self.assertIn('2', achievement_result1["new_levels"]) + self.assertIn('3', achievement_result1["new_levels"]) # Case2: NOT Achieved in first week but in next week def test_evaluate_achievement_for_weekly_evaluation_case2(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -59,8 +65,8 @@ def test_evaluate_achievement_for_weekly_evaluation_case2(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) - next_weekdate = achievement_date + datetime.timedelta(7) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement["evaluation"]) + next_weekdate = achievement_date + datetime.timedelta(11) create_achievement_user(user, achievement, achievement_date, level=1) @@ -76,17 +82,25 @@ def test_evaluate_achievement_for_weekly_evaluation_case2(self): # User has not achieved in first week but in 2nd week print("Weekly evaluation Case 2") Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) - achievement_result2 = Achievement.evaluate(user, achievement.id, achievement_date) - print(achievement_result2) + achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) + print(achievement_result) - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="weekly", dt=next_weekdate+datetime.timedelta(3)) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="weekly", dt=next_weekdate) Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) - achievement_resul1 = Achievement.evaluate(user, achievement.id, next_date) - print("achievement result1: ", achievement_resul1) + achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) + print("achievement result1: ", achievement_result1) + + self.assertEqual(achievement_result["achievement_date"], achievement_date) + self.assertEqual(achievement_result1["achievement_date"], next_date) + self.assertNotEqual(next_weekdate, next_date) + self.assertIn('1', achievement_result["levels_achieved"]) + self.assertIn('1', achievement_result1["new_levels"]) + self.assertIn('2', achievement_result1["new_levels"]) + self.assertIn('3', achievement_result1["new_levels"]) # Case3: NOT Achieved in first week but after some days in same week def test_evaluate_achievement_for_weekly_evaluation_case3(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -94,7 +108,7 @@ def test_evaluate_achievement_for_weekly_evaluation_case3(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement["evaluation"]) create_achievement_user(user, achievement, achievement_date, level=1) @@ -113,14 +127,21 @@ def test_evaluate_achievement_for_weekly_evaluation_case3(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print(achievement_result) - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="weekly", dt=achievement_date+datetime.timedelta(3)) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="weekly", dt=achievement_date+datetime.timedelta(3)) Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) - achievement_resul1 = Achievement.evaluate(user, achievement.id, next_date) - print("achievement result1: ", achievement_resul1) + achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) + print("achievement result1: ", achievement_result1) + + self.assertEqual(achievement_result["achievement_date"], achievement_date) + self.assertEqual(achievement_result1["achievement_date"], next_date) + self.assertEqual(achievement_date, next_date) + self.assertIn('1', achievement_result["levels_achieved"]) + self.assertIn('2', achievement_result1["new_levels"]) + self.assertIn('3', achievement_result1["new_levels"]) # Case1: Achieved in first and next month def test_evaluate_achievement_for_monthly_evaluation_case1(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -128,9 +149,9 @@ def test_evaluate_achievement_for_monthly_evaluation_case1(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement["evaluation"]) print(achievement_date) - next_month = achievement_date + datetime.timedelta(28) + next_month = achievement_date + datetime.timedelta(35) print(next_month) create_achievement_user(user, achievement, achievement_date, level=1) @@ -151,15 +172,25 @@ def test_evaluate_achievement_for_monthly_evaluation_case1(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="monthly", dt=next_month) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="monthly", dt=next_month) Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) - achievement_resul1 = Achievement.evaluate(user, achievement.id, next_date) - print("achievement result1: ", achievement_resul1) + achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) + print("achievement result1: ", achievement_result1) + + self.assertEqual(achievement_result["achievement_date"], achievement_date) + self.assertEqual(achievement_result1["achievement_date"], next_date) + self.assertNotEqual(next_month, next_date) + self.assertIn('1', achievement_result["levels_achieved"]) + self.assertIn('2', achievement_result["levels_achieved"]) + self.assertIn('3', achievement_result["levels_achieved"]) + self.assertIn('1', achievement_result1["new_levels"]) + self.assertIn('2', achievement_result1["new_levels"]) + self.assertIn('3', achievement_result1["new_levels"]) # Case2: Not achieved in first but in next month def test_evaluate_achievement_for_monthly_evaluation_case2(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -167,9 +198,9 @@ def test_evaluate_achievement_for_monthly_evaluation_case2(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement["evaluation"]) print(achievement_date) - next_month = achievement_date + datetime.timedelta(28) + next_month = achievement_date + datetime.timedelta(31) print(next_month) create_achievement_user(user, achievement, achievement_date, level=1) @@ -190,15 +221,23 @@ def test_evaluate_achievement_for_monthly_evaluation_case2(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="monthly", dt=next_month+datetime.timedelta(10)) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="monthly", dt=next_month+datetime.timedelta(10)) Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print("achievement result1: ", achievement_result1) - # Case2: Achieved in first month and after some days in a same month + self.assertEqual(achievement_result["achievement_date"], achievement_date) + self.assertEqual(achievement_result1["achievement_date"], next_date) + self.assertNotEqual(next_month, next_date) + self.assertIn('1', achievement_result["levels_achieved"]) + self.assertIn('1', achievement_result1["new_levels"]) + self.assertIn('2', achievement_result1["new_levels"]) + self.assertIn('3', achievement_result1["new_levels"]) + + # Case3: Achieved in first month and after some days in a same month def test_evaluate_achievement_for_monthly_evaluation_case3(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -206,7 +245,7 @@ def test_evaluate_achievement_for_monthly_evaluation_case3(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement["evaluation"]) print(achievement_date) create_achievement_user(user, achievement, achievement_date, level=1) @@ -227,14 +266,21 @@ def test_evaluate_achievement_for_monthly_evaluation_case3(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="monthly", dt=achievement_date+datetime.timedelta(10)) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="monthly", dt=achievement_date+datetime.timedelta(10)) Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print("achievement result1: ", achievement_result1) + self.assertEqual(achievement_result["achievement_date"], achievement_date) + self.assertEqual(achievement_result1["achievement_date"], next_date) + self.assertEqual(achievement_date, next_date) + self.assertIn('1', achievement_result["levels_achieved"]) + self.assertIn('2', achievement_result1["new_levels"]) + self.assertIn('3', achievement_result1["new_levels"]) + # Case1: Achieved in first year and next year def test_evaluate_achievement_for_yearly_evaluation_case1(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -242,9 +288,9 @@ def test_evaluate_achievement_for_yearly_evaluation_case1(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement["evaluation"]) print(achievement_date) - next_year = achievement_date + datetime.timedelta(365) + next_year = achievement_date + datetime.timedelta(425) print(next_year) create_achievement_user(user, achievement, achievement_date, level=1) @@ -265,15 +311,25 @@ def test_evaluate_achievement_for_yearly_evaluation_case1(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="yearly", dt=next_year) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="yearly", dt=next_year) Value.increase_value(variable_name="invite_users", user=user, value=15, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print(achievement_result1) + self.assertEqual(achievement_result["achievement_date"], achievement_date) + self.assertEqual(achievement_result1["achievement_date"], next_date) + self.assertNotEqual(next_year, next_date) + self.assertIn('1', achievement_result["levels_achieved"]) + self.assertIn('2', achievement_result["levels_achieved"]) + self.assertIn('3', achievement_result["levels_achieved"]) + self.assertIn('1', achievement_result1["new_levels"]) + self.assertIn('2', achievement_result1["new_levels"]) + self.assertIn('3', achievement_result1["new_levels"]) + # Case2: Not Achieved in first year but in next year def test_evaluate_achievement_for_yearly_evaluation_case2(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -281,9 +337,9 @@ def test_evaluate_achievement_for_yearly_evaluation_case2(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement["evaluation"]) print(achievement_date) - next_year = achievement_date + datetime.timedelta(365) + next_year = achievement_date + datetime.timedelta(534) print(next_year) create_achievement_user(user, achievement, achievement_date, level=1) @@ -304,15 +360,23 @@ def test_evaluate_achievement_for_yearly_evaluation_case2(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="yearly", dt=next_year + datetime.timedelta(10)) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="yearly", dt=next_year + datetime.timedelta(10)) Value.increase_value(variable_name="invite_users", user=user, value=15, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) print("achievement result1: ", achievement_result1) + self.assertEqual(achievement_result["achievement_date"], achievement_date) + self.assertEqual(achievement_result1["achievement_date"], next_date) + self.assertNotEqual(next_year, next_date) + self.assertIn('1', achievement_result["levels_achieved"]) + self.assertIn('1', achievement_result1["new_levels"]) + self.assertIn('2', achievement_result1["new_levels"]) + self.assertIn('3', achievement_result1["new_levels"]) + # Case3: Achieved in this year and after some days in same year def test_evaluate_achievement_for_yearly_evaluation_case3(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3, @@ -320,9 +384,9 @@ def test_evaluate_achievement_for_yearly_evaluation_case3(self): user = create_user() - achievement_date = Achievement.get_datetime_for_evaluation_type(user["timezone"], achievement["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement["evaluation"]) print(achievement_date) - next_year = achievement_date + datetime.timedelta(365) + next_year = achievement_date + datetime.timedelta(501) print(next_year) create_achievement_user(user, achievement, achievement_date, level=1) @@ -343,8 +407,15 @@ def test_evaluate_achievement_for_yearly_evaluation_case3(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - next_date = Achievement.get_datetime_for_evaluation_type(tz=user.timezone, evaluation_type="yearly", dt=achievement_date + datetime.timedelta(110)) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="yearly", dt=achievement_date + datetime.timedelta(110)) Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) - print("achievement result1: ", achievement_result1) \ No newline at end of file + print("achievement result1: ", achievement_result1) + + self.assertEqual(achievement_result["achievement_date"], achievement_date) + self.assertEqual(achievement_result1["achievement_date"], next_date) + self.assertEqual(achievement_date, next_date) + self.assertIn('1', achievement_result["levels_achieved"]) + self.assertIn('2', achievement_result1["new_levels"]) + self.assertIn('3', achievement_result1["new_levels"]) \ No newline at end of file diff --git a/gengine/app/tests/test_auth.py b/gengine/app/tests/test_auth.py index cbb3d93..60f543e 100644 --- a/gengine/app/tests/test_auth.py +++ b/gengine/app/tests/test_auth.py @@ -61,10 +61,6 @@ def test_user_updation(self): self.assertTrue(user.timezone == "Europe/Bukarest") self.assertTrue(user.language_id == lang.id) - # Failing cases - self.assertTrue(user.additional_public_data["first_name"] == "Matthew") - self.assertTrue(user.additional_public_data["last_name"] == "Hayden") - def test_user_deletion(self): user1 = create_user() @@ -91,10 +87,7 @@ def test_user_deletion(self): # Correct cases self.assertNotIn(user1.id, remaining_users) - self.assertIn(user2.id, remaining_users) - - # Failing cases - self.assertNotIn(user2.id, remaining_users) + self.assertEqual(user2.id, remaining_users[0].id) def test_verify_password(self): auth_user = AuthUser() diff --git a/gengine/app/tests/test_device.py b/gengine/app/tests/test_device.py index f9fa38c..81b3100 100644 --- a/gengine/app/tests/test_device.py +++ b/gengine/app/tests/test_device.py @@ -41,10 +41,6 @@ def test_update_user_device(self): self.assertTrue(device.push_id == '5126') self.assertTrue(device.app_version == '1.2') - # Failing cases - self.assertTrue(device.device_os == 'Android') - - diff --git a/gengine/app/tests/test_eval_types_and_rewards.py b/gengine/app/tests/test_eval_types_and_rewards.py index 6d51849..f97f63c 100644 --- a/gengine/app/tests/test_eval_types_and_rewards.py +++ b/gengine/app/tests/test_eval_types_and_rewards.py @@ -1,14 +1,12 @@ import datetime -import pytz from gengine.app.cache import clear_all_caches from gengine.app.tests.base import BaseDBTest -from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_value, create_goals, create_achievement_rewards, create_achievement_user, create_goal_evaluation_cache -from gengine.metadata import DBSession -from gengine.app.model import Achievement, AchievementUser, Value, t_values -from gengine.base.model import update_connection +from gengine.app.tests.helpers import create_user, create_achievement, create_variable, create_goals, create_achievement_user +from gengine.app.model import Achievement, Value -class TestAchievementEvaluationTypeAndRewards(BaseDBTest): + +class TestEvaluationForMultipleUsersAndTimzone(BaseDBTest): def test_multiple_users_achievemnt_reward(self): @@ -77,7 +75,7 @@ def test_multiple_users_achievemnt_reward(self): create_achievement_user(user4, achievement, achievement_date1, level=1) create_variable("invite_users", variable_group="day") - Value.increase_value(variable_name="invite_users", user=user1, value=3, key=None) + Value.increase_value(variable_name="invite_users", user=user1, value=12, key=None) Value.increase_value(variable_name="invite_users", user=user2, value=2, key=None) Value.increase_value(variable_name="invite_users", user=user3, value=11, key=None) Value.increase_value(variable_name="invite_users", user=user4, value=6, key=None) @@ -90,12 +88,31 @@ def test_multiple_users_achievemnt_reward(self): clear_all_caches() + print("test for multiple users") # Evaluate achievement for all users achievement1 = Achievement.evaluate(user3, achievement.id, achievement_date1) - print(achievement1) + print(achievement1["goals"][1]["leaderboard"]) new_date = achievement_date1+datetime.timedelta(7) print(new_date) next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation, dt=new_date) achievement = Achievement.evaluate(user3, achievement.id, next_date) - print(achievement) + print(achievement["goals"][1]["leaderboard"][0]) + + self.assertEqual(3, achievement1["goals"][1]["leaderboard"][0]["user"]["id"]) + self.assertEqual(1, achievement1["goals"][1]["leaderboard"][1]["user"]["id"]) + self.assertEqual(2, achievement1["goals"][1]["leaderboard"][2]["user"]["id"]) + + self.assertEqual(9.0, achievement1["goals"][1]["leaderboard"][0]["value"]) + self.assertEqual(6.0, achievement1["goals"][1]["leaderboard"][1]["value"]) + self.assertEqual(2.0, achievement1["goals"][1]["leaderboard"][2]["value"]) + + self.assertEqual(1, achievement["goals"][1]["leaderboard"][0]["user"]["id"]) + self.assertEqual(2, achievement["goals"][1]["leaderboard"][1]["user"]["id"]) + self.assertEqual(3, achievement["goals"][1]["leaderboard"][2]["user"]["id"]) + + self.assertEqual(6.0, achievement["goals"][1]["leaderboard"][0]["value"]) + self.assertEqual(2.0, achievement["goals"][1]["leaderboard"][1]["value"]) + self.assertEqual(0.0, achievement["goals"][1]["leaderboard"][2]["value"]) + + diff --git a/gengine/app/tests/test_goal.py b/gengine/app/tests/test_goal.py index fa59686..51b0b22 100644 --- a/gengine/app/tests/test_goal.py +++ b/gengine/app/tests/test_goal.py @@ -5,7 +5,7 @@ class TestEvaluateGoal(BaseDBTest): def test_compute_progress(self): - return + user = create_user() create_variable(variable_name="invite_users", variable_group="day") Value.increase_value(variable_name="invite_users", user=user, value=6, key=None) @@ -20,7 +20,8 @@ def test_compute_progress(self): goal = create_goals(achievement) # goal is for invite_users, its group_by_key is false, progress is sum of all the values - users_progress_goal = Goal.compute_progress(goal, achievement, user.id) + achievement_date = Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement["evaluation"]) + users_progress_goal = Goal.compute_progress(goal=goal, achievement=achievement, user=user, evaluation_date=achievement_date) goal_evaluation = {e["user_id"]: e["value"] for e in users_progress_goal} print(goal_evaluation) @@ -29,7 +30,8 @@ def test_compute_progress(self): # For goal1, since its group_by_key is True, it'll add the values of the same key achievement1 = create_achievement(achievement_name="participate_achievement") goal1 = create_goals(achievement1) - users_progress_goal1 = Goal.compute_progress(goal1, achievement, user.id) + achievement_date1= Achievement.get_datetime_for_evaluation_type(User.get_user(user.id)["timezone"], achievement1["evaluation"]) + users_progress_goal1 = Goal.compute_progress(goal=goal1, achievement=achievement1, user=user, evaluation_date=achievement_date1) goal_evaluation1 = {e["user_id"]: e["value"] for e in users_progress_goal1} print(goal_evaluation1) @@ -37,16 +39,13 @@ def test_compute_progress(self): # Check with group_by_key for goals participate = False goal2 = create_goals(achievement1, goal_group_by_key=False) - users_progress_goal1 = Goal.compute_progress(goal2, achievement, user.id) + users_progress_goal1 = Goal.compute_progress(goal=goal2, achievement=achievement1, user=user, evaluation_date=achievement_date1) goal_evaluation2 = {e["user_id"]: e["value"] for e in users_progress_goal1} print(goal_evaluation2) self.assertLessEqual(goal_evaluation2.get(user.id), 10) - # If group_by_key attribute for goal is not set, then default value is considered as False and NOT None - # In compute_progress function , group_by_key is compared with None. Is it desired or need to change it to False? - def test_evaluate_goal(self): - return + user = create_user() create_variable(variable_name="invite_users", variable_group="day") Value.increase_value(variable_name="invite_users", user=user, value=6, key=None) @@ -72,9 +71,9 @@ def test_evaluate_goal(self): goal2 = create_goals(achievement, goal_group_by_key=True, goal_goal="3*level") evaluation_result2 = Goal.evaluate(goal2, achievement, achievement_date, user.id, level=4, goal_eval_cache_before=False, execute_triggers=True) print(evaluation_result2) - # failing cases - self.assertGreaterEqual(evaluation_result2["value"], 12) - self.assertEqual(evaluation_result2["achieved"], True) + + self.assertLessEqual(evaluation_result2["value"], 12) + self.assertEqual(evaluation_result2["achieved"], False) # Goal invite_users achievement1 = create_achievement(achievement_name="invite_users_achievement") @@ -84,12 +83,11 @@ def test_evaluate_goal(self): evaluation_result1 = Goal.evaluate(goal1, achievement1, achievement_date1, user.id, level=2, goal_eval_cache_before=False, execute_triggers=True) print(evaluation_result1) - # True cases self.assertGreaterEqual(evaluation_result1["value"], 8) self.assertEqual(evaluation_result1["achieved"], True) def test_get_goal_properties(self): - return + achievement = create_achievement() goals = create_goals(achievement) @@ -102,14 +100,11 @@ def test_get_goal_properties(self): result1 = Goal.get_goal_properties(goals.id, level1) print(result1) - # True test self.assertIsNot(result, []) - - # False test - self.assertNotEquals(result1, []) + self.assertEquals(result1, []) def test_get_leaderboard(self): - return + achievement = create_achievement(achievement_name="invite_users_achievement") goals = create_goals(achievement) @@ -180,7 +175,8 @@ def test_get_leaderboard(self): positions = Goal.get_leaderboard(goals, achievement_date_for_user3, user_ids) print(positions) self.assertEqual(positions[0]["value"], 22.00) - self.assertEqual(positions[0]["value"], 8.00) + self.assertEqual(positions[1]["value"], 15.00) + self.assertEqual(positions[2]["value"], 8.00) # Test for Goal is not evaluated for few user_ids create_variable(variable_name="invite_users", variable_group="day") @@ -192,5 +188,3 @@ def test_get_leaderboard(self): print(positions) self.assertEqual(positions[0]["value"], 15.00) - - #Should the leaderbord be chosen from users whose goal_achieved is True?? \ No newline at end of file From 7df0bd94ca5e91cf0a4cd9065eeebe654a086afa Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Fri, 24 Feb 2017 18:07:51 +0100 Subject: [PATCH 138/176] fix generation of achievement dates --- gengine/app/views.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/gengine/app/views.py b/gengine/app/views.py index 1a9b9d4..f838bcb 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -166,20 +166,17 @@ def may_view(achievement, requesting_user): achievement["evaluation"], dt=d ) - if dr == None: - achievement_dates.add(None) - else: - while d<=now: - achievement_dates.add(dr) - + achievement_dates.add(dr) + if dr != None: + while d <= now: if achievement["evaluation"] == "yearly": - d += datetime.timedelta(days=365) + d += datetime.timedelta(days=364) elif achievement["evaluation"] == "monthly": d += datetime.timedelta(days=28) elif achievement["evaluation"] == "weekly": - d += datetime.timedelta(days=7) + d += datetime.timedelta(days=6) elif achievement["evaluation"] == "daily": - d += datetime.timedelta(days=1) + d += datetime.timedelta(hours=23) else: break # should not happen @@ -189,6 +186,9 @@ def may_view(achievement, requesting_user): dt=d ) + if dr <= now: + achievement_dates.add(dr) + i=0 for achievement_date in reversed(sorted(achievement_dates)): # We execute the goal triggers only for the newest and previous period, not for any periods longer ago From fa863a29d43c61d0f92227d769eefa640285972a Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Fri, 24 Feb 2017 18:18:38 +0100 Subject: [PATCH 139/176] fix None key storage as NULL --- gengine/app/model.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 72c5610..0610336 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -731,20 +731,22 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): variable = Variable.get_variable_by_name(variable_name) dt = Variable.get_datetime_for_tz_and_group(tz,variable["group"],at_datetime=at_datetime) - condition = and_(t_values.c.datetime==dt, - t_values.c.variable_id==variable["id"], - t_values.c.user_id==user_id, - t_values.c.key==str(key)) + key = None if key is None else str(key) + + condition = and_(t_values.c.datetime == dt, + t_values.c.variable_id == variable["id"], + t_values.c.user_id == user_id, + t_values.c.key == key) current_value = DBSession.execute(select([t_values.c.value,]).where(condition)).scalar() if current_value is not None: update_connection().execute(t_values.update(condition, values={"value":current_value+value})) else: - update_connection().execute(t_values.insert({"datetime":dt, - "variable_id":variable["id"], - "user_id" : user_id, - "key" : str(key), - "value":value})) + update_connection().execute(t_values.insert({"datetime": dt, + "variable_id": variable["id"], + "user_id": user_id, + "key": key, + "value": value})) Variable.invalidate_caches_for_variable_and_user(variable["id"],user["id"]) new_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() From 607eadbc34ef5c8f86b33ecb345dba3925a5683b Mon Sep 17 00:00:00 2001 From: Nilakshi Naphade Date: Sat, 25 Feb 2017 23:24:00 +0100 Subject: [PATCH 140/176] Adding Fixing issues in achievement tests. Adding correct assert statements --- gengine/app/tests/helpers.py | 1 - gengine/app/tests/test_achievement1.py | 261 ++++++++++--------------- 2 files changed, 102 insertions(+), 160 deletions(-) diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index fc06769..b05a6bc 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -77,7 +77,6 @@ def create_user( if user_id is undefined: user_id = (DBSession.execute("SELECT max(id) as c FROM users").scalar() or 0) + 1 - print() if lat is undefined: lat = randrange_float(gen_data["area"]["min_lat"],gen_data["area"]["max_lat"]) diff --git a/gengine/app/tests/test_achievement1.py b/gengine/app/tests/test_achievement1.py index ff9a8c4..2ff1534 100644 --- a/gengine/app/tests/test_achievement1.py +++ b/gengine/app/tests/test_achievement1.py @@ -115,7 +115,7 @@ def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): "first_name": "Michael", "last_name": "Clarke" }, - friends = [1, 4] + friends = [user1.id] ) # Create Third user @@ -131,7 +131,7 @@ def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): "first_name": "Rudolf", "last_name": "Red Nose" }, - friends=[1, 2] + friends=[user1.id, user2.id] ) # Create Fourth user @@ -147,7 +147,7 @@ def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): "first_name": "Steve", "last_name": "Waugh" }, - friends=[2, 3] + friends=[user2.id, user3.id] ) achievement = create_achievement() @@ -156,34 +156,26 @@ def test_get_relevant_users_by_achievement_friends_and_user_reverse(self): usersForFriend3 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user3.id) usersForFriend4 = achievement.get_relevant_users_by_achievement_and_user_reverse(achievement, user4.id) - print(usersForFriend1) - print(usersForFriend2) - print(usersForFriend3) - print(usersForFriend4) - - self.assertIn(2, usersForFriend1) - # self.assertIn(3, usersForFriend1) - self.assertIn(3, usersForFriend2) - self.assertIn(4, usersForFriend2) - self.assertIn(4, usersForFriend3) - self.assertIn(1, usersForFriend4) - self.assertIn(4, usersForFriend4) + self.assertIn(user2.id, usersForFriend1) + self.assertIn(user3.id, usersForFriend1) + self.assertIn(user3.id, usersForFriend2) + self.assertIn(user4.id, usersForFriend2) + self.assertIn(user4.id, usersForFriend3) + self.assertIn(user4.id, usersForFriend4) def test_get_level(self): user = create_user(timezone="Australia/Sydney", country="Australia", region="xyz", city="Sydney") achievement = create_achievement(achievement_name="invite_users_achievement", achievement_evaluation="weekly") - achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation) + achievement_date = Achievement.get_datetime_for_evaluation_type(evaluation_timezone=achievement.evaluation_timezone, evaluation_type="weekly") create_achievement_user(user, achievement, achievement_date, level=2) achievement.get_level(user.id, achievement["id"], achievement_date) level = achievement.get_level_int(user.id, achievement.id, achievement_date) - achievement_date1 = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement_date+datetime.timedelta(7)) - print("achievement date") - print(achievement_date1) + achievement_date1 = Achievement.get_datetime_for_evaluation_type(evaluation_timezone=achievement.evaluation_timezone, evaluation_type="weekly", dt=achievement_date + datetime.timedelta(7)) achievement.get_level(user.id, achievement["id"], achievement_date1) level1 = achievement.get_level_int(user.id, achievement.id, achievement_date1) @@ -193,33 +185,27 @@ def test_get_level(self): self.assertEqual(level, 2) self.assertEqual(level1, 0) - # Failed cases - self.assertEqual(level1, 1) - def test_get_rewards(self): - return - achievement = create_achievement() - create_achievement_rewards(achievement) + achievement = create_achievement(achievement_maxlevel=3) + create_achievement_rewards(achievement) + clear_all_caches() rewardlist1 = Achievement.get_rewards(achievement.id, 1) - print(rewardlist1) + print("rewardlist1",rewardlist1) - rewardlist2 = Achievement.get_rewards(achievement.id, 2) - print(rewardlist2) + rewardlist2 = Achievement.get_rewards(achievement.id, 5) + print("rewardlist2", rewardlist2) rewardlist3 = Achievement.get_rewards(achievement.id, 3) - print(rewardlist3) - - # passed test cases - self.assertNotEqual(rewardlist2, []) - self.assertNotEqual(rewardlist3, []) + print("rewardlist3", rewardlist3) - # failed test cases self.assertEqual(rewardlist1, []) + self.assertEqual(rewardlist2, []) + self.assertNotEqual(rewardlist3, []) def test_get_achievement_properties(self): - return - achievement = create_achievement() + + achievement = create_achievement(achievement_maxlevel=3) achievementproperty = AchievementProperty() achievementproperty.name = "xp" @@ -234,6 +220,8 @@ def test_get_achievement_properties(self): DBSession.add(achievements_achievementproperty) DBSession.flush() + clear_all_caches() + result1 = Achievement.get_achievement_properties(achievement.id, 4) print(result1) @@ -241,10 +229,10 @@ def test_get_achievement_properties(self): print(result2) self.assertNotEqual(result1, []) - self.assertNotEqual(result2, []) + self.assertEqual(result2, []) def test_evaluate_achievement_for_participate(self): - return + achievement = create_achievement(achievement_name="participate_achievement", achievement_relevance="own", achievement_maxlevel=4) user = create_user() @@ -263,76 +251,68 @@ def test_evaluate_achievement_for_participate(self): variable = create_variable("participate", variable_group="day") Value.increase_value(variable_name=variable.name, user=user, value=1, key="5") - goal = create_goals(achievement, - goal_condition="""{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""", - goal_group_by_key=True, - goal_operator="geq", - goal_goal="1*level") + create_goals(achievement, + goal_condition="""{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate"}}""", + goal_group_by_key=True, + goal_operator="geq", + goal_goal="1*level") + + clear_all_caches() level = Achievement.evaluate(user, achievement.id, achievement_date).get("level") - print("level ",level) - result1 = Value.increase_value(variable_name="participate", user=user, value=5, key="5") - print("value result ",result1) - level2 = Achievement.evaluate(user, achievement.id, achievement_date) - print("level2 ",level2) - result2 = Value.increase_value(variable_name="participate", user=user, value=1, key="5") - # result3 = Value.increase_value(variable_name="participate", user=user, value=1, key="5") - # result = Achievement.evaluate(user, achievement.id, achievement_date) - #self.assertEqual(result["level"], achievement.maxlevel) + Value.increase_value(variable_name="participate", user=user, value=1, key="7") + level2 = Achievement.evaluate(user, achievement.id, achievement_date).get("level") + + Value.increase_value(variable_name="participate", user=user, value=5, key="5") + level1 = Achievement.evaluate(user, achievement.id, achievement_date).get("level") + + self.assertEqual(level, 1) + self.assertEqual(level2, 1) + self.assertEqual(level1, 4) def test_evaluate_achievement_for_invite_users(self): - return - # Achievement with relevance friends and maxlevel 3 - achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3) + + achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=10) user = create_user() achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation) - # get level - current_level = 1 - achievement_user = AchievementUser() - achievement_user.user_id = user.id - achievement_user.achievement_id = achievement.id - achievement_user.achievement_date = achievement_date - achievement_user.level = current_level - DBSession.add(achievement_user) - DBSession.flush() + create_achievement_user(user=user, achievement=achievement, achievement_date=achievement_date, level=1) update_connection().execute(t_values.delete()) create_variable("invite_users", variable_group="day") - firstvalue = Value.increase_value(variable_name="invite_users", user=user, value=1, key=None) - print("firstvalue ", firstvalue) + Value.increase_value(variable_name="invite_users", user=user, value=1, key=None) create_goals(achievement, - goal_goal="3*level", + goal_goal="1*level", goal_operator="geq", goal_group_by_key=False ) clear_all_caches() - achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) - #print("achievement result: ",achievement_result) + level = Achievement.evaluate(user, achievement.id, achievement_date).get("level") + print("level: ", level) - new2 = Value.increase_value(variable_name="invite_users", user=user, value=8, key=None) - print("new2 ",new2) - achievement_result1 = Achievement.evaluate(user, achievement.id, achievement_date) - #print(achievement_result1) + Value.increase_value(variable_name="invite_users", user=user, value=8, key=None) + level1 = Achievement.evaluate(user, achievement.id, achievement_date).get("level") + print("level1 ", level1) - new3 = Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) - print(new3) - achievement_result2 = Achievement.evaluate(user, achievement.id, achievement_date) - #print("achievement_result2: ", achievement_result2) + Value.increase_value(variable_name="invite_users", user=user, value=5, key=None) + level2 = Achievement.evaluate(user, achievement.id, achievement_date).get("level") + print("level2: ", level2) + self.assertEqual(level, 1) + self.assertEqual(level1, 9) + self.assertEqual(level2, 10) def test_get_reward_and_properties_for_achievement(self): - return + user = create_user() achievement = create_achievement(achievement_name="invite_users_achievement", achievement_relevance="friends", achievement_maxlevel=3) - # Check for property achievementproperty = AchievementProperty() achievementproperty.name = "xp" DBSession.add(achievementproperty) @@ -346,109 +326,65 @@ def test_get_reward_and_properties_for_achievement(self): DBSession.add(achievements_achievementproperty) DBSession.flush() - reward = Reward() - reward.name = "badge" - DBSession.add(reward) - DBSession.flush() - - achievement_reward = AchievementReward() - achievement_reward.achievement_id = achievement.id - achievement_reward.reward_id = reward.id - achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level1}.png" - achievement_reward.from_level = 2 - DBSession.add(achievement_reward) - DBSession.flush() + create_achievement_rewards(achievement=achievement) achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation) - current_level = 1 - achievement_user = AchievementUser() - achievement_user.user_id = user.id - achievement_user.achievement_id = achievement.id - achievement_user.achievement_date = achievement_date - achievement_user.level = current_level - DBSession.add(achievement_user) - DBSession.flush() + create_achievement_user(user=user, achievement=achievement, achievement_date=achievement_date, level=1) - variable = create_variable("invite_users", "none") - firstvalue = Value.increase_value(variable_name="invite_users", user=user, value=2, key="5") - DBSession.flush() + create_variable("invite_users", "none") + Value.increase_value(variable_name="invite_users", user=user, value=4, key="5") - goal = create_goals(achievement = achievement, - goal_condition="""{"term": {"type": "literal", "variable": "invite_users"}}""", - goal_group_by_key=True, - goal_operator="geq", - goal_goal="1*level") + create_goals(achievement = achievement, + goal_condition="""{"term": {"type": "literal", "variable": "invite_users"}}""", + goal_group_by_key=True, + goal_operator="geq", + goal_goal="1*level") + clear_all_caches() result = Achievement.evaluate(user, achievement.id, achievement_date) - print(result) + print("reward_achievement_result:",result) - self.assertNotEqual(len(result["new_levels"]["2"]["rewards"]), 0) - - Value.increase_value(variable_name="invite_users", user=user, value=2, key="5") - DBSession.flush() - - # result = Achievement.evaluate(user, achievement.id, achievement_date) - # print(result) - - # result1 = Achievement.get_achievement_properties(achievement.id, 4) - # print("in test property",result1) - - self.assertNotEqual(len(result["new_levels"]["2"]["properties"]), 0) + self.assertEqual(len(result["new_levels"]["2"]["rewards"]), 0) + self.assertEqual(len(result["new_levels"]["3"]["rewards"]), 1) + self.assertEqual(len(result["new_levels"]["2"]["properties"]), 1) + self.assertEqual(len(result["new_levels"]["3"]["properties"]), 1) def test_multiple_goals_of_same_achievement(self): - return + user = create_user() achievement = create_achievement(achievement_name="participate_achievement", achievement_maxlevel=3) - reward = Reward() - reward.name = "badge" - DBSession.add(reward) - DBSession.flush() - - achievement_reward = cre - achievement_reward = AchievementReward() - achievement_reward.achievement_id = achievement.id - achievement_reward.reward_id = reward.id - achievement_reward.value = "https://www.gamification-software.com/img/trophy_{level1}.png" - achievement_reward.from_level = 2 - DBSession.add(achievement_reward) - DBSession.flush() + create_achievement_rewards(achievement=achievement) achievement_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation) - goal1 = create_goals(achievement=achievement, - goal_condition="""{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate_seminar"}}""", - goal_group_by_key=False, - goal_operator="geq", - goal_goal="2*level", - goal_name = "goal_participate_seminar") + create_goals(achievement=achievement, + goal_condition="""{"term": {"key": ["5","7"], "type": "literal", "key_operator": "IN", "variable": "participate_seminar"}}""", + goal_group_by_key=False, + goal_operator="geq", + goal_goal="2*level", + goal_name = "goal_participate_seminar") - goal2 = create_goals(achievement=achievement, - goal_condition="""{"term": {"type": "literal", "variable": "participate_talk"}}""", - goal_group_by_key=False, - goal_operator="geq", - goal_goal="1*level", - goal_name="goal_participate_talk") + create_goals(achievement=achievement, + goal_condition="""{"term": {"type": "literal", "variable": "participate_talk"}}""", + goal_group_by_key=False, + goal_operator="geq", + goal_goal="1*level", + goal_name="goal_participate_talk") - current_level = 1 - achievement_user = AchievementUser() - achievement_user.user_id = user.id - achievement_user.achievement_id = achievement.id - achievement_user.achievement_date = achievement_date - achievement_user.level = current_level - DBSession.add(achievement_user) - DBSession.flush() + clear_all_caches() + create_achievement_user(user=user, achievement=achievement, achievement_date=achievement_date, level=1) - variable1 = create_variable("participate_seminar",variable_group=None) + variable1 = create_variable("participate_seminar", variable_group=None) variable2 = create_variable("participate_talk", variable_group=None) Value.increase_value(variable1.name, user, "2", "5") - Value.increase_value(variable1.name, user, "2", "7") - Value.increase_value(variable2.name, user, "2", key=None) + Value.increase_value(variable1.name, user, "3", "7") + Value.increase_value(variable2.name, user, "3", key=None) result = Achievement.evaluate(user, achievement.id, achievement_date) - print(result) + print("multiple_goals_of_same_achievement:",result) Value.increase_value(variable1.name, user, "2", "7") result1 = Achievement.evaluate(user, achievement.id, achievement_date) print(result1) @@ -456,4 +392,11 @@ def test_multiple_goals_of_same_achievement(self): result2 = Achievement.evaluate(user, achievement.id, achievement_date) print(result2) - self.assertNotEqual(len(result["new_levels"]["2"]["rewards"]), 0) \ No newline at end of file + self.assertEqual(len(result["levels"]["3"]["rewards"]), 1) + self.assertEqual(result["levels"]["1"]["goals"]["1"]["goal_goal"], 2) + self.assertEqual(result["levels"]["3"]["goals"]["2"]["goal_goal"], 3) + self.assertEqual(result1["levels"]["2"]["goals"]["1"]["goal_goal"], 4) + self.assertEqual(result1["levels"]["3"]["goals"]["2"]["goal_goal"], 3) + self.assertEqual(result2["levels"]["2"]["goals"]["1"]["goal_goal"], 4) + self.assertEqual(result2["levels"]["3"]["goals"]["2"]["goal_goal"], 3) + From 340a67ec5aecaf705f1e8ec8f98fb05f5da4adc1 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Thu, 2 Mar 2017 21:07:49 +0100 Subject: [PATCH 141/176] Declare encoding for non-ascii chars and py27 --- gengine/app/tests/test_achievement1.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gengine/app/tests/test_achievement1.py b/gengine/app/tests/test_achievement1.py index 2ff1534..4703a95 100644 --- a/gengine/app/tests/test_achievement1.py +++ b/gengine/app/tests/test_achievement1.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import datetime import pytz From 29e0e7bd0e353ebc487a87e4fa3d9b0ebcd7987c Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Thu, 2 Mar 2017 21:11:10 +0100 Subject: [PATCH 142/176] Update test_auth.py --- gengine/app/tests/test_auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gengine/app/tests/test_auth.py b/gengine/app/tests/test_auth.py index 60f543e..9d7436d 100644 --- a/gengine/app/tests/test_auth.py +++ b/gengine/app/tests/test_auth.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from gengine.app.tests.base import BaseDBTest from gengine.app.tests.helpers import create_user, update_user, delete_user, get_or_create_language from gengine.metadata import DBSession From 32f3123cc347d2641614b7c2ad5b8439b5a60ca6 Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Fri, 3 Mar 2017 13:25:53 +0100 Subject: [PATCH 143/176] Use github tarball --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a16601d..684ba0b 100644 --- a/setup.py +++ b/setup.py @@ -83,6 +83,6 @@ gengine = gengine:redgalaxy """, dependency_links=[ - "https://github.com/ActiDoo/PyAPNs/archive/master.zip#egg=apns", + "https://github.com/ActiDoo/PyAPNs/tarball/master#egg=apns", ], ) From 392acee132dd27e736e574822133d515dc840746 Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Sat, 4 Mar 2017 00:03:15 +0100 Subject: [PATCH 144/176] load environment variables before first use of settings --- gengine/__init__.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/gengine/__init__.py b/gengine/__init__.py index eac577f..b179f92 100644 --- a/gengine/__init__.py +++ b/gengine/__init__.py @@ -21,6 +21,14 @@ def main(global_config, **settings): """ This function returns a Pyramid WSGI application. """ + + durl = os.environ.get("DATABASE_URL") #heroku + if durl: + settings['sqlalchemy.url']=durl + + murl = os.environ.get("MEMCACHED_URL") #heroku + if murl: + settings['urlcache_url']=murl set_settings(settings) @@ -43,15 +51,6 @@ def reset_context_on_new_request(event): reset_context() config.add_subscriber(reset_context_on_new_request,NewRequest) config.include('pyramid_dogpile_cache') - - durl = os.environ.get("DATABASE_URL") #heroku - if durl: - settings['sqlalchemy.url']=durl - - murl = os.environ.get("MEMCACHED_URL") #heroku - if murl: - settings['urlcache_url']=murl - config.include("pyramid_tm") config.include('pyramid_chameleon') @@ -120,4 +119,4 @@ def datetime_adapter(obj, request): config.scan() - return HTTPSProxied(config.make_wsgi_app()) \ No newline at end of file + return HTTPSProxied(config.make_wsgi_app()) From 54165cf8482efb575dec1a99ab2f95b968507935 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Tue, 7 Mar 2017 11:40:09 +0100 Subject: [PATCH 145/176] use empty string for None key --- gengine/app/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 0610336..eaabe4b 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -729,9 +729,9 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): tz = user["timezone"] variable = Variable.get_variable_by_name(variable_name) - dt = Variable.get_datetime_for_tz_and_group(tz,variable["group"],at_datetime=at_datetime) + dt = Variable.get_datetime_for_tz_and_group(tz, variable["group"], at_datetime=at_datetime) - key = None if key is None else str(key) + key = '' if key is None else str(key) condition = and_(t_values.c.datetime == dt, t_values.c.variable_id == variable["id"], From 4cf762d830209fccea77fb4ae1b27a0c23f45d9e Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Tue, 7 Mar 2017 11:53:40 +0100 Subject: [PATCH 146/176] tests: delete all caches before each test --- gengine/app/tests/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gengine/app/tests/base.py b/gengine/app/tests/base.py index dec3e6b..32f0e2c 100644 --- a/gengine/app/tests/base.py +++ b/gengine/app/tests/base.py @@ -15,6 +15,8 @@ def setUpClass(cls): super(BaseDBTest, cls).setUpClass() def setUp(self): + from gengine.app.cache import clear_all_caches + clear_all_caches() self.db = db.db() dsn = self.db.dsn() self.engine = create_engine( From 4bcfe9af3cae2140aafd3ffb5e4ec6a9d22b8c95 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Tue, 7 Mar 2017 12:11:20 +0100 Subject: [PATCH 147/176] add test for None key, set runner to fail fast, fix two assertions --- gengine/app/tests/runner.py | 2 +- .../app/tests/test_achievement_integration_tests.py | 4 ++-- gengine/app/tests/test_value.py | 13 ++++++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/gengine/app/tests/runner.py b/gengine/app/tests/runner.py index 686c860..ab57fd8 100644 --- a/gengine/app/tests/runner.py +++ b/gengine/app/tests/runner.py @@ -34,7 +34,7 @@ def create_test_suite(): db.setupDB() testSuite = create_test_suite() - text_runner = unittest.TextTestRunner().run(testSuite) + text_runner = unittest.TextTestRunner(failfast=True).run(testSuite) finally: try: db.unsetupDB() diff --git a/gengine/app/tests/test_achievement_integration_tests.py b/gengine/app/tests/test_achievement_integration_tests.py index d3947f4..d03d0f4 100644 --- a/gengine/app/tests/test_achievement_integration_tests.py +++ b/gengine/app/tests/test_achievement_integration_tests.py @@ -221,7 +221,7 @@ def test_evaluate_achievement_for_monthly_evaluation_case2(self): achievement_result = Achievement.evaluate(user, achievement.id, achievement_date) print("achievement result: ", achievement_result) - next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="monthly", dt=next_month+datetime.timedelta(10)) + next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, evaluation_type="monthly", dt=next_month+datetime.timedelta(days=10)) Value.increase_value(variable_name="invite_users", user=user, value=10, key=None, at_datetime=next_date) achievement_result1 = Achievement.evaluate(user, achievement.id, next_date) @@ -229,7 +229,7 @@ def test_evaluate_achievement_for_monthly_evaluation_case2(self): self.assertEqual(achievement_result["achievement_date"], achievement_date) self.assertEqual(achievement_result1["achievement_date"], next_date) - self.assertNotEqual(next_month, next_date) + self.assertGreaterEqual(next_month, next_date) # next_month can be the 1st, 2nd, 3rd of 4th (February) self.assertIn('1', achievement_result["levels_achieved"]) self.assertIn('1', achievement_result1["new_levels"]) self.assertIn('2', achievement_result1["new_levels"]) diff --git a/gengine/app/tests/test_value.py b/gengine/app/tests/test_value.py index 0623ce8..c9932a4 100644 --- a/gengine/app/tests/test_value.py +++ b/gengine/app/tests/test_value.py @@ -5,7 +5,6 @@ class TestValue(BaseDBTest): def test_increase_value(self): - return user = create_user() variable = create_variable(variable_name="participate", variable_group="day") @@ -15,9 +14,13 @@ def test_increase_value(self): # Correct cases self.assertGreater(value2, value1) - - # Failing cases - self.assertGreater(value3, value2) + self.assertEqual(value3, value2) # Doesn't work when give variable_group = none i.e. current_datetime check which differes for two successive calls - # Increase value is being called only in evaluate_achievement function and not in evaluate_goal \ No newline at end of file + # Increase value is being called only in evaluate_achievement function and not in evaluate_goal + + def test_increase_value_null_key(self): + user = create_user() + variable = create_variable(variable_name="login", variable_group="day") + value1 = Value.increase_value(variable.name, user, value=1, key=None) + self.assertIs(value1, 1) From c1ffb0754293e597e68f19fa63e3c31a891703bf Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Tue, 7 Mar 2017 12:34:00 +0100 Subject: [PATCH 148/176] fix leaderboard query for recurring achievements --- gengine/app/model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index eaabe4b..56b8655 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1544,6 +1544,7 @@ def get_leaderboard(cls, goal, achievement_date, user_ids): t_goal_evaluation_cache.c.value])\ .where(and_(t_goal_evaluation_cache.c.user_id.in_(user_ids), t_goal_evaluation_cache.c.goal_id==goal["id"], + t_goal_evaluation_cache.c.achievement_date==achievement_date, ))\ .order_by(t_goal_evaluation_cache.c.value.desc(), t_goal_evaluation_cache.c.user_id.desc()) @@ -1551,7 +1552,9 @@ def get_leaderboard(cls, goal, achievement_date, user_ids): users = User.get_users(user_ids) - missing_user_ids = set(user_ids)-set([x["user_id"] for x in items]) + requested_user_ids = set(int(s) for s in user_ids) + values_found_for_user_ids = set([int(x["user_id"]) for x in items]) + missing_user_ids = requested_user_ids - values_found_for_user_ids missing_users = User.get_users(missing_user_ids).values() if len(missing_users)>0: #the goal has not been evaluated for some users... From 4a06dee483b3796dc06f65076735c08c9b83388b Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Tue, 7 Mar 2017 13:03:59 +0100 Subject: [PATCH 149/176] fix friends leaderboard test --- .../app/tests/test_eval_types_and_rewards.py | 49 ++++++------------- 1 file changed, 16 insertions(+), 33 deletions(-) diff --git a/gengine/app/tests/test_eval_types_and_rewards.py b/gengine/app/tests/test_eval_types_and_rewards.py index f97f63c..2d338bc 100644 --- a/gengine/app/tests/test_eval_types_and_rewards.py +++ b/gengine/app/tests/test_eval_types_and_rewards.py @@ -8,7 +8,7 @@ class TestEvaluationForMultipleUsersAndTimzone(BaseDBTest): - def test_multiple_users_achievemnt_reward(self): + def test_friends_leaderboard(self): user1 = create_user() @@ -69,50 +69,33 @@ def test_multiple_users_achievemnt_reward(self): print("Achievement date for first user:") print(achievement_date1) - create_achievement_user(user1, achievement, achievement_date1, level=1) - create_achievement_user(user2, achievement, achievement_date1, level=1) - create_achievement_user(user3, achievement, achievement_date1, level=1) - create_achievement_user(user4, achievement, achievement_date1, level=1) - create_variable("invite_users", variable_group="day") - Value.increase_value(variable_name="invite_users", user=user1, value=12, key=None) - Value.increase_value(variable_name="invite_users", user=user2, value=2, key=None) - Value.increase_value(variable_name="invite_users", user=user3, value=11, key=None) - Value.increase_value(variable_name="invite_users", user=user4, value=6, key=None) create_goals(achievement, - goal_goal="3*level", + goal_goal=None, goal_operator="geq", goal_group_by_key=False - ) + ) + + Value.increase_value(variable_name="invite_users", user=user1, value=12, key=None) + Value.increase_value(variable_name="invite_users", user=user2, value=2, key=None) + Value.increase_value(variable_name="invite_users", user=user3, value=11, key=None) + Value.increase_value(variable_name="invite_users", user=user4, value=6, key=None) clear_all_caches() print("test for multiple users") - # Evaluate achievement for all users + + # Evaluate achievement for friends of user 3 achievement1 = Achievement.evaluate(user3, achievement.id, achievement_date1) print(achievement1["goals"][1]["leaderboard"]) - new_date = achievement_date1+datetime.timedelta(7) - print(new_date) - next_date = Achievement.get_datetime_for_evaluation_type(achievement.evaluation_timezone, achievement.evaluation, dt=new_date) - achievement = Achievement.evaluate(user3, achievement.id, next_date) - print(achievement["goals"][1]["leaderboard"][0]) - - self.assertEqual(3, achievement1["goals"][1]["leaderboard"][0]["user"]["id"]) - self.assertEqual(1, achievement1["goals"][1]["leaderboard"][1]["user"]["id"]) - self.assertEqual(2, achievement1["goals"][1]["leaderboard"][2]["user"]["id"]) + # user 3 has to friends: user 1 and user 2 + self.assertEqual(user1["id"], achievement1["goals"][1]["leaderboard"][0]["user"]["id"]) + self.assertEqual(user3["id"], achievement1["goals"][1]["leaderboard"][1]["user"]["id"]) + self.assertEqual(user2["id"], achievement1["goals"][1]["leaderboard"][2]["user"]["id"]) - self.assertEqual(9.0, achievement1["goals"][1]["leaderboard"][0]["value"]) - self.assertEqual(6.0, achievement1["goals"][1]["leaderboard"][1]["value"]) + self.assertEqual(12.0, achievement1["goals"][1]["leaderboard"][0]["value"]) + self.assertEqual(11.0, achievement1["goals"][1]["leaderboard"][1]["value"]) self.assertEqual(2.0, achievement1["goals"][1]["leaderboard"][2]["value"]) - self.assertEqual(1, achievement["goals"][1]["leaderboard"][0]["user"]["id"]) - self.assertEqual(2, achievement["goals"][1]["leaderboard"][1]["user"]["id"]) - self.assertEqual(3, achievement["goals"][1]["leaderboard"][2]["user"]["id"]) - - self.assertEqual(6.0, achievement["goals"][1]["leaderboard"][0]["value"]) - self.assertEqual(2.0, achievement["goals"][1]["leaderboard"][1]["value"]) - self.assertEqual(0.0, achievement["goals"][1]["leaderboard"][2]["value"]) - - From c25b79d4607c111f7b3903dcec768bd9cb5ba04f Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 8 Mar 2017 18:37:11 +0100 Subject: [PATCH 150/176] move push delivery to script, must be executed via cronjob --- .../versions/2012674516fc_has_been_pushed.py | 30 +++++++ gengine/app/model.py | 20 +++-- gengine/maintenance/scripts/push_messages.py | 79 +++++++++++++++++++ setup.py | 1 + 4 files changed, 123 insertions(+), 7 deletions(-) create mode 100644 gengine/app/alembic/versions/2012674516fc_has_been_pushed.py create mode 100644 gengine/maintenance/scripts/push_messages.py diff --git a/gengine/app/alembic/versions/2012674516fc_has_been_pushed.py b/gengine/app/alembic/versions/2012674516fc_has_been_pushed.py new file mode 100644 index 0000000..4543881 --- /dev/null +++ b/gengine/app/alembic/versions/2012674516fc_has_been_pushed.py @@ -0,0 +1,30 @@ +"""has_been_pushed + +Revision ID: 2012674516fc +Revises: 62026366cd60 +Create Date: 2017-03-08 17:44:02.214248 + +""" + +# revision identifiers, used by Alembic. +revision = '2012674516fc' +down_revision = '62026366cd60' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('user_messages', sa.Column('has_been_pushed', sa.Boolean(), nullable=False, server_default='1')) + op.create_index(op.f('ix_user_messages_has_been_pushed'), 'user_messages', ['has_been_pushed'], unique=False) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_user_messages_has_been_pushed'), table_name='user_messages') + op.drop_column('user_messages', 'has_been_pushed') + ### end Alembic commands ### diff --git a/gengine/app/model.py b/gengine/app/model.py index 56b8655..98843fa 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -298,6 +298,7 @@ def get_default_token_valid_time(): Column('translation_id', ty.Integer, ForeignKey("translationvariables.id", ondelete="RESTRICT"), nullable = True), Column('params', JSON(), nullable=True, default={}), Column('is_read', ty.Boolean, index=True, default=False, nullable=False), + Column('has_been_pushed', ty.Boolean, index=True, default=True, server_default='0', nullable=False), Column('created_at', ty.DateTime(), nullable=False, default=datetime.datetime.utcnow, index=True), ) @@ -1688,7 +1689,6 @@ def text(self): @classmethod def deliver(cls, message): from gengine.app.push import send_push_message - text = UserMessage.get_text(message) language = get_settings().get("fallback_language", "en") j = t_users.join(t_languages) @@ -1697,12 +1697,18 @@ def deliver(cls, message): language = user_language["name"] translated_text = text[language] - send_push_message( - user_id = message["user_id"], - text = translated_text, - custom_payload = {}, - title = get_settings().get("push_title","Gamification-Engine") - ) + if not message["has_been_pushed"]: + try: + send_push_message( + user_id = message["user_id"], + text = translated_text, + custom_payload = {}, + title = get_settings().get("push_title","Gamification-Engine") + ) + except Exception as e: + log.error(e, exc_info=True) + else: + DBSession.execute(t_user_messages.update().values({ "has_been_pushed" : True }).where(t_user_messages.c.id == message["id"])) class GoalTrigger(ABase): def __unicode__(self, *args, **kwargs): diff --git a/gengine/maintenance/scripts/push_messages.py b/gengine/maintenance/scripts/push_messages.py new file mode 100644 index 0000000..620e858 --- /dev/null +++ b/gengine/maintenance/scripts/push_messages.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +import sys +import logging + +from zope.sqlalchemy.datamanager import mark_changed + +from gengine.metadata import MySession + +log = logging.getLogger(__name__) +log.addHandler(logging.StreamHandler()) + +import os +import pyramid_dogpile_cache +import transaction +from gengine.app.cache import init_caches +from pyramid.config import Configurator +from pyramid.paster import ( + get_appsettings, + setup_logging, +) +from pyramid.scripts.common import parse_vars +from sqlalchemy import engine_from_config + +def usage(argv): + cmd = os.path.basename(argv[0]) + print('usage: %s [var=value]\n' + '(example: "%s production.ini")' % (cmd, cmd)) + sys.exit(1) + + +def main(argv=sys.argv): + if len(argv) < 2: + usage(argv) + config_uri = argv[1] + options = parse_vars(argv[2:]) + setup_logging(config_uri) + settings = get_appsettings(config_uri, options=options) + + from gengine.base.settings import set_settings + set_settings(settings) + + durl = os.environ.get("DATABASE_URL") # heroku + if durl: + settings['sqlalchemy.url'] = durl + + murl = os.environ.get("MEMCACHED_URL") + if murl: + settings['urlcache_url'] = murl + + engine = engine_from_config(settings, 'sqlalchemy.') + + config = Configurator(settings=settings) + pyramid_dogpile_cache.includeme(config) + + from gengine.metadata import ( + init_session, + init_declarative_base, + init_db + ) + init_session() + init_declarative_base() + init_db(engine) + init_caches() + + from gengine.metadata import ( + DBSession + ) + sess = DBSession() + init_session(override_session=sess, replace=True) + + import gengine.app.model as m + with transaction.manager: + mark_changed(sess, transaction.manager, True) + + messages = sess.execute(m.t_user_messages.select().where(m.t_user_messages.c.has_been_pushed == False)) + for msg in messages: + m.UserMessage.deliver(msg) + sess.flush() + sess.commit() diff --git a/setup.py b/setup.py index 684ba0b..414524e 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,7 @@ gengine_quickstart = gengine.maintenance.scripts.quickstart:main generate_gengine_erd = gengine.maintenance.scripts.generate_erd:main generate_gengine_revision = gengine.maintenance.scripts.generate_revision:main + gengine_push_messages = gengine.maintenance.scripts.push_messages:main [redgalaxy.plugins] gengine = gengine:redgalaxy """, From 63ffd712382b4206b6583461435491138d8780c0 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 8 Mar 2017 18:37:43 +0100 Subject: [PATCH 151/176] check user_id in get_progress --- gengine/app/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gengine/app/views.py b/gengine/app/views.py index 525b6a4..2aa25c4 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -213,7 +213,10 @@ def may_view(achievement, requesting_user): @view_config(route_name='get_progress', renderer='json', request_method="GET") def get_progress(request): """get all relevant data concerning the user's progress""" - user_id = int(request.matchdict["user_id"]) + try: + user_id = int(request.matchdict["user_id"]) + except: + raise APIError(400, "illegal_user_id", "no valid user_id given") user = User.get_user(user_id) if not user: From ca8407cdd29dba610b183ca7ed6ffccabf2206f9 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 8 Mar 2017 18:41:01 +0100 Subject: [PATCH 152/176] remove message delivery from trigger execution --- gengine/app/model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 98843fa..3c73d7d 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1740,7 +1740,6 @@ def execute(cls, trigger_step, user_id, current_percentage, value, goal_goal, go is_read = False, ) uS.add(m) - UserMessage.deliver(m) @event.listens_for(GoalTriggerStep, "after_insert") @event.listens_for(GoalTriggerStep, 'after_update') From 4f2f0ecf39d48486b1bb983ee3bfd21f94e389e6 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 8 Mar 2017 18:59:29 +0100 Subject: [PATCH 153/176] set has been pushed to false when creating a new msg --- gengine/app/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gengine/app/model.py b/gengine/app/model.py index 3c73d7d..b0b2b5a 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -1738,6 +1738,7 @@ def execute(cls, trigger_step, user_id, current_percentage, value, goal_goal, go 'percentage' : current_percentage },**goal_properties), is_read = False, + has_been_pushed = False ) uS.add(m) From b4f4272688e2329b9bc7ed46975d0cdd51b76b71 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 8 Mar 2017 19:04:07 +0100 Subject: [PATCH 154/176] fix update_connection for cron --- gengine/base/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/base/model.py b/gengine/base/model.py index 0ef8f4d..cfed947 100644 --- a/gengine/base/model.py +++ b/gengine/base/model.py @@ -112,6 +112,6 @@ def valid_timezone(timezone): def update_connection(): - session = meta.DBSession() + session = meta.DBSession() if callable(meta.DBSession) else meta.DBSession mark_changed(session) return session From 0fcd8b1a76baeef3834b393f521a042c9b759307 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 8 Mar 2017 19:19:24 +0100 Subject: [PATCH 155/176] fix push code session handling --- gengine/app/push.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gengine/app/push.py b/gengine/app/push.py index e8879b4..f92b7d2 100644 --- a/gengine/app/push.py +++ b/gengine/app/push.py @@ -118,7 +118,6 @@ def gcm_feedback(response): for reg_id, success_id in response['success'].items(): log.debug('Successfully sent notification for reg_id {0}'.format(reg_id)) - uS = update_connection() # Handling errors if 'errors' in response: @@ -129,7 +128,7 @@ def gcm_feedback(response): # Remove reg_ids from database for reg_id in reg_ids: q = t_user_device.delete().where(t_user_device.c.push_id == reg_id) - uS.execute(q) + DBSession.execute(q) # Repace reg_id with canonical_id in your database if 'canonical' in response: @@ -143,7 +142,9 @@ def gcm_feedback(response): "push_id" : canonical_id }).where(t_user_device.c.push_id == reg_id) - uS.execute(q) + DBSession.execute(q) + + DBSession.flush() def send_push_message( user_id, From c40ce7f9e058de767c90bedded03559138d4776d Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 8 Mar 2017 20:03:35 +0100 Subject: [PATCH 156/176] respect variable date when clearing evaluation cache --- gengine/app/model.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index b0b2b5a..990c768 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -685,17 +685,17 @@ def map_variables_to_rules(cls): return m @classmethod - def invalidate_caches_for_variable_and_user(cls,variable_id,user_id): + def invalidate_caches_for_variable_and_user(cls, variable_id, user_id, dt): """ invalidate the relevant caches for this user and all relevant users with concerned leaderboards""" - goalsandachievements = cls.map_variables_to_rules().get(variable_id,[]) + goalsandachievements = cls.map_variables_to_rules().get(variable_id, []) Goal.clear_goal_caches(user_id, [ - (entry["goal"]["id"], Achievement.get_datetime_for_evaluation_type(entry["achievement"]["evaluation_timezone"], entry["achievement"]["evaluation"])) + (entry["goal"]["id"], Achievement.get_datetime_for_evaluation_type(entry["achievement"]["evaluation_timezone"], entry["achievement"]["evaluation"], dt=dt)) for entry in goalsandachievements ] ) for entry in goalsandachievements: - achievement_date = Achievement.get_datetime_for_evaluation_type(entry["achievement"]["evaluation_timezone"], entry["achievement"]["evaluation"]) + achievement_date = Achievement.get_datetime_for_evaluation_type(entry["achievement"]["evaluation_timezone"], entry["achievement"]["evaluation"],dt=dt) Achievement.invalidate_evaluate_cache(user_id, entry["achievement"], achievement_date) @classmethod @@ -749,7 +749,7 @@ def increase_value(cls, variable_name, user, value, key, at_datetime=None): "key": key, "value": value})) - Variable.invalidate_caches_for_variable_and_user(variable["id"],user["id"]) + Variable.invalidate_caches_for_variable_and_user(variable_id=variable["id"], user_id=user["id"], dt = dt) new_value = DBSession.execute(select([t_values.c.value, ]).where(condition)).scalar() return new_value From ae399f2e77fe4819edfb79492aafc6d45c21bf04 Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Thu, 16 Mar 2017 15:47:02 +0100 Subject: [PATCH 157/176] Update PostgreSQL version to 9.6 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3bf56d4..5578aa5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,11 @@ python: - "3.5" sudo: required env: - - TEST_POSTGRES=/usr/lib/postgresql/9.5/bin/postgres TEST_INITDB=/usr/lib/postgresql/9.5/bin/initdb + - TEST_POSTGRES=/usr/lib/postgresql/9.6/bin/postgres TEST_INITDB=/usr/lib/postgresql/9.6/bin/initdb apt: packages: - - postgresql-9.5 - - postgresql-contrib-9.5 + - postgresql-9.6 + - postgresql-contrib-9.6 - postgis # command to install dependencies install: From 7f4d75a205bc5e2c3a75794a33180b8ffe59ed58 Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Thu, 16 Mar 2017 16:00:27 +0100 Subject: [PATCH 158/176] Fix test runner (return 1 if test failed) --- gengine/app/tests/runner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gengine/app/tests/runner.py b/gengine/app/tests/runner.py index ab57fd8..94852ab 100644 --- a/gengine/app/tests/runner.py +++ b/gengine/app/tests/runner.py @@ -22,6 +22,7 @@ def create_test_suite(): return suite if __name__=="__main__": + exit = 1 try: redis = testing.redis.RedisServer() @@ -35,6 +36,8 @@ def create_test_suite(): db.setupDB() testSuite = create_test_suite() text_runner = unittest.TextTestRunner(failfast=True).run(testSuite) + if text_runner.wasSuccessful(): + exit = 0 finally: try: db.unsetupDB() @@ -44,3 +47,4 @@ def create_test_suite(): redis.stop() except: log.exception() + sys.exit(exit) From a6ae4171de33dd77e9109523380c1330d4037f9f Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Thu, 16 Mar 2017 16:04:21 +0100 Subject: [PATCH 159/176] Add missing import for sys --- gengine/app/tests/runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gengine/app/tests/runner.py b/gengine/app/tests/runner.py index 94852ab..0a4307a 100644 --- a/gengine/app/tests/runner.py +++ b/gengine/app/tests/runner.py @@ -5,6 +5,7 @@ import pkgutil import testing.redis import logging +import sys log = logging.getLogger(__name__) From fceabad9bb0e2a1ecd98d39d86f9180b579f9689 Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Thu, 16 Mar 2017 16:11:23 +0100 Subject: [PATCH 160/176] Add email notifications about build status --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5578aa5..4f5350e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,3 +31,10 @@ deploy: password: $PYPI_PASSWORD on: tags: true + +notifications: + email: + recipients: + - $ADMIN_EMAIL + on_success: always # default: change + on_failure: always # default: always From fd068b45241a1f41311b97652af2d12e89a54793 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 22 Mar 2017 15:39:22 +0100 Subject: [PATCH 161/176] pull apns from github --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9e87478..ca81850 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ alembic==0.8.8 -apns==2.0.1 +-e git+https://github.com/ActiDoo/PyAPNs@2bd5517c791333188c949c61d239a63d1596d539#egg=apns argon2==0.1.10 Chameleon==2.25 click==6.6 From 3d6187f355e941bbe6f73de809eb196390c74ed0 Mon Sep 17 00:00:00 2001 From: Marcel Sander Date: Wed, 22 Mar 2017 15:39:44 +0100 Subject: [PATCH 162/176] argon2 needs longer salt --- gengine/app/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gengine/app/model.py b/gengine/app/model.py index 990c768..9025e5a 100644 --- a/gengine/app/model.py +++ b/gengine/app/model.py @@ -347,7 +347,7 @@ def password(self,new_pw): import argon2 import crypt import base64 - self.password_salt = crypt.mksalt() + self.password_salt = crypt.mksalt()+crypt.mksalt()+crypt.mksalt()+crypt.mksalt()+crypt.mksalt() hash = argon2.argon2_hash(new_pw, self.password_salt) self.password_hash = base64.b64encode(hash).decode("UTF-8") From 9bd097841261d50f6c2794c1a6cb62db176e27e7 Mon Sep 17 00:00:00 2001 From: Sander <-> Date: Tue, 28 Mar 2017 09:39:54 +0200 Subject: [PATCH 163/176] set apns dependency to temporary python 3 fork --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ca81850..59edaa2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ alembic==0.8.8 --e git+https://github.com/ActiDoo/PyAPNs@2bd5517c791333188c949c61d239a63d1596d539#egg=apns +tapns3==3.0.0 argon2==0.1.10 Chameleon==2.25 click==6.6 From d5f8fbc607dab4d43a0d1b95cf6ab75c323cbca9 Mon Sep 17 00:00:00 2001 From: Sander <-> Date: Tue, 28 Mar 2017 09:41:58 +0200 Subject: [PATCH 164/176] set apns dependency to temporary python 3 fork --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 414524e..029f418 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ 'jsonschema', 'pyparsing', 'argon2', - 'apns', + 'tapns3', 'python-gcm', #Testing: 'testing.postgresql', From bef82fede9985e5f7624a2b45e8db5f33f8082a8 Mon Sep 17 00:00:00 2001 From: Sander <-> Date: Tue, 28 Mar 2017 09:42:13 +0200 Subject: [PATCH 165/176] modify ini files to match new features --- development.ini | 13 ++++++++++++- gengine_quickstart_template/production.ini | 20 +++++++++++++++++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/development.ini b/development.ini index 5de6c5d..135e621 100644 --- a/development.ini +++ b/development.ini @@ -32,7 +32,7 @@ flaskadmin_secret = 87ghsjkdjfhg85grsfgsdfghwez89hsuif # replace with random str # dogpile cache dogpile_cache.backend = dogpile.cache.null -dogpile_incache.general.backend = dogpile.cache.dbm +dogpile_cache.general.backend = dogpile.cache.dbm dogpile_cache.general.arguments.filename = general.dbm dogpile_cache.achievement_eval.backend = dogpile.cache.dbm @@ -59,6 +59,17 @@ urlcache_active = false # callback url, will be used for time-related leaderboard evaluations (daily,monthly,yearly) (TBD) notify_progress = +enable_user_authentication = false +fallback_language = en +gcm.api_key= +gcm.package= +apns.dev.key= +apns.dev.certificate= +apns.prod.key= +apns.prod.certificate= +push_title=Gamification Engine + + ### # wsgi server configuration ### diff --git a/gengine_quickstart_template/production.ini b/gengine_quickstart_template/production.ini index 165ec87..49fcc8b 100644 --- a/gengine_quickstart_template/production.ini +++ b/gengine_quickstart_template/production.ini @@ -24,9 +24,6 @@ urlcacheid = gengine #flaskadmin settings flaskadmin_secret = 87ghsjkdjfhg85grsfgsdfghwez89hsuif # replace with random string! -force_https = false -urlprefix = - # dogpile cache dogpile_cache.backend = dogpile.cache.null @@ -42,6 +39,13 @@ dogpile_cache.achievements_by_user_for_today.arguments.filename = achievements_b dogpile_cache.translations.backend = dogpile.cache.dbm dogpile_cache.translations.arguments.filename = translations.dbm +dogpile_cache.achievements_users_levels.backend = dogpile.cache.dbm +dogpile_cache.achievements_users_levels.arguments.filename = achievements_users_levels.dbm + +dogpile_cache.goal_evaluation.backend = dogpile.cache.dbm +dogpile_cache.goal_evaluation.arguments.filename = goal_evaluation.dbm + +dogpile_cache.goal_statements.backend = dogpile.cache.memory # memcache urlcache_url = 127.0.0.1:11211 @@ -50,6 +54,16 @@ urlcache_active = true # callback url, will be used for time-related leaderboard evaluations (daily,monthly,yearly) (TBD) notify_progress = +enable_user_authentication = false +fallback_language = en +gcm.api_key= +gcm.package= +apns.dev.key= +apns.dev.certificate= +apns.prod.key= +apns.prod.certificate= +push_title=Gamification Engine + ### # wsgi server configuration ### From 950764e11e8fce6e32e6c0c3e8e14b9820177cdc Mon Sep 17 00:00:00 2001 From: Sander <-> Date: Tue, 28 Mar 2017 11:02:49 +0200 Subject: [PATCH 166/176] prepare 0.2 (mainly docs) --- CHANGES.txt | 9 +- development.ini | 2 +- docs/_static/2017-03-28-erm.svg | 880 ++++++++++++++++++++++++++++++++ docs/concepts/index.rst | 39 +- docs/conf.py | 8 +- docs/internals/index.rst | 14 +- docs/rest/index.rst | 9 +- docs/roadmap.rst | 12 +- docs/upgrading/index.rst | 18 + gengine/app/formular.py | 14 +- gengine/app/views.py | 2 +- setup.py | 2 +- 12 files changed, 966 insertions(+), 43 deletions(-) create mode 100644 docs/_static/2017-03-28-erm.svg create mode 100644 docs/upgrading/index.rst diff --git a/CHANGES.txt b/CHANGES.txt index 945c9b4..1348bd0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1 +1,8 @@ -. \ No newline at end of file +0.2 + * Implement new rule syntax + * Add time-aware / recurring achievements + * Add optional authentication & authorization + * Introduce Goal Triggers + * Introduce Mobile Pushes + * Introduce Messages + * Lots of bugfixes diff --git a/development.ini b/development.ini index 135e621..64ba63d 100644 --- a/development.ini +++ b/development.ini @@ -15,7 +15,7 @@ pyramid.includes = pyramid_debugtoolbar pyramid_tm -sqlalchemy.url = postgres://nilakshi@127.0.0.1/gengine +sqlalchemy.url = postgres://dev:dev@127.0.0.1/gengine #reverse proxy settings force_https = false diff --git a/docs/_static/2017-03-28-erm.svg b/docs/_static/2017-03-28-erm.svg new file mode 100644 index 0000000..0e26a4b --- /dev/null +++ b/docs/_static/2017-03-28-erm.svg @@ -0,0 +1,880 @@ + + + + + + +%3 + + +users + +users + +id + [BIGINT] + +lat + [DOUBLE PRECISION] + +lng + [DOUBLE PRECISION] + +language_id + [INTEGER] + +timezone + [VARCHAR] + +country + [VARCHAR] + +region + [VARCHAR] + +city + [VARCHAR] + +additional_public_data + [JSON] + +created_at + [TIMESTAMP WITHOUT TIME ZONE] + + +achievements_users + +achievements_users + +id + [INTEGER] + +user_id + [BIGINT] + +achievement_id + [INTEGER] + +achievement_date + [TIMESTAMP WITHOUT TIME ZONE] + +level + [INTEGER] + +updated_at + [TIMESTAMP WITHOUT TIME ZONE] + + +users--achievements_users + +{0,1} +0..N + + +goal_trigger_executions + +goal_trigger_executions + +id + [BIGINT] + +trigger_step_id + [INTEGER] + +user_id + [BIGINT] + +execution_level + [INTEGER] + +execution_date + [TIMESTAMP WITHOUT TIME ZONE] + +achievement_date + [TIMESTAMP WITHOUT TIME ZONE] + + +users--goal_trigger_executions + +{0,1} +0..N + + +goal_evaluation_cache + +goal_evaluation_cache + +id + [INTEGER] + +goal_id + [INTEGER] + +achievement_date + [TIMESTAMP WITHOUT TIME ZONE] + +user_id + [BIGINT] + +achieved + [BOOLEAN] + +value + [DOUBLE PRECISION] + + +users--goal_evaluation_cache + +{0,1} +0..N + + +user_devices + +user_devices + +device_id + [VARCHAR(255)] + +user_id + [BIGINT] + +device_os + [VARCHAR] + +push_id + [VARCHAR(255)] + +app_version + [VARCHAR(255)] + +registered_at + [TIMESTAMP WITHOUT TIME ZONE] + + +users--user_devices + +{0,1} +0..N + + +users_groups + +users_groups + +user_id + [BIGINT] + +group_id + [BIGINT] + + +users--users_groups + +{0,1} +0..N + + +users_users + +users_users + +from_id + [BIGINT] + +to_id + [BIGINT] + + +users--users_users + +{0,1} +0..N + + +users--users_users + +{0,1} +0..N + + +auth_users + +auth_users + +user_id + [BIGINT] + +email + [VARCHAR] + +password_hash + [VARCHAR] + +password_salt + [VARCHAR] + +active + [BOOLEAN] + +created_at + [TIMESTAMP WITHOUT TIME ZONE] + + +users--auth_users + +{0,1} +0..N + + +user_messages + +user_messages + +id + [BIGINT] + +user_id + [BIGINT] + +translation_id + [INTEGER] + +params + [JSON] + +is_read + [BOOLEAN] + +has_been_pushed + [BOOLEAN] + +created_at + [TIMESTAMP WITHOUT TIME ZONE] + + +users--user_messages + +{0,1} +0..N + + +values + +values + +user_id + [BIGINT] + +datetime + [TIMESTAMP WITH TIME ZONE] + +variable_id + [INTEGER] + +value + [INTEGER] + +key + [VARCHAR(100)] + + +users--values + +{0,1} +0..N + + +auth_tokens + +auth_tokens + +id + [BIGINT] + +user_id + [BIGINT] + +token + [VARCHAR] + +valid_until + [TIMESTAMP WITHOUT TIME ZONE] + + +auth_users--auth_tokens + +{0,1} +0..N + + +auth_users_roles + +auth_users_roles + +user_id + [BIGINT] + +role_id + [BIGINT] + + +auth_users--auth_users_roles + +{0,1} +0..N + + +goals_goalproperties + +goals_goalproperties + +goal_id + [INTEGER] + +property_id + [INTEGER] + +value + [VARCHAR(255)] + +value_translation_id + [INTEGER] + +from_level + [INTEGER] + + +groups + +groups + +id + [BIGINT] + + +groups--users_groups + +{0,1} +0..N + + +achievementproperties + +achievementproperties + +id + [INTEGER] + +name + [VARCHAR(255)] + +is_variable + [BOOLEAN] + + +achievements_achievementproperties + +achievements_achievementproperties + +achievement_id + [INTEGER] + +property_id + [INTEGER] + +value + [VARCHAR(255)] + +value_translation_id + [INTEGER] + +from_level + [INTEGER] + + +achievementproperties--achievements_achievementproperties + +{0,1} +0..N + + +achievementcategories + +achievementcategories + +id + [INTEGER] + +name + [VARCHAR(255)] + + +achievements + +achievements + +id + [INTEGER] + +achievementcategory_id + [INTEGER] + +name + [VARCHAR(255)] + +maxlevel + [INTEGER] + +hidden + [BOOLEAN] + +valid_start + [DATE] + +valid_end + [DATE] + +lat + [DOUBLE PRECISION] + +lng + [DOUBLE PRECISION] + +max_distance + [INTEGER] + +priority + [INTEGER] + +evaluation + [evaluation_types] + +evaluation_timezone + [VARCHAR] + +relevance + [relevance_types] + +view_permission + [achievement_view_permission] + +created_at + [TIMESTAMP WITHOUT TIME ZONE] + + +achievementcategories--achievements + +{0,1} +0..N + + +achievements--achievements_users + +{0,1} +0..N + + +achievements--achievements_achievementproperties + +{0,1} +0..N + + +requirements + +requirements + +from_id + [INTEGER] + +to_id + [INTEGER] + + +achievements--requirements + +{0,1} +0..N + + +achievements--requirements + +{0,1} +0..N + + +denials + +denials + +from_id + [INTEGER] + +to_id + [INTEGER] + + +achievements--denials + +{0,1} +0..N + + +achievements--denials + +{0,1} +0..N + + +goals + +goals + +id + [INTEGER] + +name + [VARCHAR(255)] + +name_translation_id + [INTEGER] + +condition + [VARCHAR(255)] + +timespan + [INTEGER] + +group_by_key + [BOOLEAN] + +group_by_dateformat + [VARCHAR(255)] + +goal + [VARCHAR(255)] + +operator + [goal_operators] + +maxmin + [goal_maxmin] + +achievement_id + [INTEGER] + +priority + [INTEGER] + + +achievements--goals + +{0,1} +0..N + + +achievements_rewards + +achievements_rewards + +id + [INTEGER] + +achievement_id + [INTEGER] + +reward_id + [INTEGER] + +value + [VARCHAR(255)] + +value_translation_id + [INTEGER] + +from_level + [INTEGER] + + +achievements--achievements_rewards + +{0,1} +0..N + + +goal_trigger_steps + +goal_trigger_steps + +id + [INTEGER] + +goal_trigger_id + [INTEGER] + +step + [INTEGER] + +condition_type + [goal_trigger_condition_types] + +condition_percentage + [DOUBLE PRECISION] + +action_type + [goal_trigger_action_types] + +action_translation_id + [INTEGER] + + +goal_trigger_steps--goal_trigger_executions + +{0,1} +0..N + + +auth_roles_permissions + +auth_roles_permissions + +id + [INTEGER] + +role_id + [INTEGER] + +name + [VARCHAR(255)] + + +goals--goal_evaluation_cache + +{0,1} +0..N + + +goals--goals_goalproperties + +{0,1} +0..N + + +goal_triggers + +goal_triggers + +id + [INTEGER] + +name + [VARCHAR(100)] + +goal_id + [INTEGER] + +execute_when_complete + [BOOLEAN] + + +goals--goal_triggers + +{0,1} +0..N + + +goal_triggers--goal_trigger_steps + +{0,1} +0..N + + +rewards + +rewards + +id + [INTEGER] + +name + [VARCHAR(255)] + + +rewards--achievements_rewards + +{0,1} +0..N + + +translations + +translations + +id + [INTEGER] + +translationvariable_id + [INTEGER] + +language_id + [INTEGER] + +text + [TEXT] + + +variables + +variables + +id + [INTEGER] + +name + [VARCHAR(255)] + +group + [variable_group_types] + +increase_permission + [variable_increase_permission] + + +variables--values + +{0,1} +0..N + + +languages + +languages + +id + [INTEGER] + +name + [VARCHAR(255)] + + +languages--users + +{0,1} +0..N + + +languages--translations + +{0,1} +0..N + + +alembic_version + +alembic_version + +version_num + [VARCHAR(32)] + + +auth_roles + +auth_roles + +id + [INTEGER] + +name + [VARCHAR(100)] + + +auth_roles--auth_roles_permissions + +{0,1} +0..N + + +auth_roles--auth_users_roles + +{0,1} +0..N + + +goalproperties + +goalproperties + +id + [INTEGER] + +name + [VARCHAR(255)] + +is_variable + [BOOLEAN] + + +goalproperties--goals_goalproperties + +{0,1} +0..N + + +translationvariables + +translationvariables + +id + [INTEGER] + +name + [VARCHAR(255)] + + +translationvariables--user_messages + +{0,1} +0..N + + +translationvariables--goals_goalproperties + +{0,1} +0..N + + +translationvariables--achievements_achievementproperties + +{0,1} +0..N + + +translationvariables--goal_trigger_steps + +{0,1} +0..N + + +translationvariables--goals + +{0,1} +0..N + + +translationvariables--achievements_rewards + +{0,1} +0..N + + +translationvariables--translations + +{0,1} +0..N + + + diff --git a/docs/concepts/index.rst b/docs/concepts/index.rst index ab4d40c..c13bf82 100644 --- a/docs/concepts/index.rst +++ b/docs/concepts/index.rst @@ -39,7 +39,7 @@ Goals define conditions that need to be fulfilled in order to get an achievement - goal: the value that is used for comparison - operator: "geq" or "leq"; used for comparison - - condition: the rule as python code, see below + - condition: the rule in json format, see below - group_by_dateformat: passed as a parameter to to_char ( PostgreSQL-Docs_ ) e.g. you can select and group by the weekday by using "ID" for ISO 8601 day of the week (1-7) which can afterwards be used in the condition - group_by_key: group by the key of the values table @@ -58,9 +58,16 @@ We first need to create a variable "participate" and tell our application to inc The constraint that a user may not attend multiple times to one seminar is covered by the application and not discussed here. In the gamification-engine we create a Goal with the following formular: -.. code:: python +.. code:: json - and_(p.var=="participate", p.key.in_(["5","7","9"])) + { + "term": { + "type": "literal", + "variable": "participate", + "key": ["5","7","9"], + "key_operator": "IN" + } + } Whenever a value for "participate" is set, this Goal is evaluated. It sums up all rows with the given condition and compares it to the Goal's "goal" attribute using the given operator. @@ -73,13 +80,19 @@ We create a variable "invite_users" and set the condition as follows: .. code:: python p.var=="invite_users" + { + "term": { + "type": "literal", + "variable": "invite_users" + } + } Furthermore we set the Goal's goal to 30 and the operator to "geq". If you want to make use of Goals with multiple levels, you probably want to increase the goal attribute with every level. -Therefore, you can also use python formulars. +Therefore, you can mathematical formulas. Example: @@ -87,7 +100,9 @@ For the first level, the user needs to invite 5 other users, for the second leve .. code:: python - 5*p.level # p.level is set by the gamification engine + 5*level # level is set by the gamification engine + +For further information about the rule language, we currently need to refer to .. _the sources: https://github.com/ActiDoo/gamification-engine/blob/develop/gengine/app/formular.py . Achievements ============ @@ -98,18 +113,22 @@ To allow multiple levels, you can set the *maxlevel* attribute. You can specify time-based constraints by setting *valid_start* and *valid_end*, and location-based constraints by setting *lat*,*lng* and *max_distance*. -The *hidden* flag can be used to model secret achievements. The *priority* specifies a custom order in output lists. +The *hidden* flag can be used to model secret achievements. The *priority* specifies a custom order in output lists. Achievements can also be used to model leaderboards. Therefor you need to assign a single Goal whose *goal attribute* is set to None. The Achievement's *relevance* attribute specifies in which context the leaderboard should be computed. Valid values are "friends", "city" and "own". +For setting up recurring achievements, set the *evaluation* to e.g. *monthly*. The *evaluation_timezone* parameter specifies when exactly the periods begin and end. + +There is a *view_permission* setting that can be used when authorization is active. It specifies whether other users can see the goal progress. + Properties ========== A property describes Achievements or Goals of our system, like the name, image, description or XP the user should get. -The Values of Properties can again be python formulars. -Inside the formular you can make use of the level by using *p.level*. +The Values of Properties can again be python formulas. +Inside the formula you can make use of the level by using *level*. Additionally Properties can be used as Variables. This is useful to model goals like "reach 1000xp". @@ -119,8 +138,8 @@ Rewards ======= From the model perspective Rewards are similar to Properties. The main difference occurs during the evaluation of Achievements, more specifically when a user reaches a new level. -While the formulars for the properties are simply evaluated for the specific level, -the evaluated formulars of the rewards are compared to lower levels. +While the formulas for the properties are simply evaluated for the specific level, +the evaluated formulas of the rewards are compared to lower levels. The engine thus knows for each achieved level, which reward is new and can tell the application about this. In your application this could for example trigger a badge notification. diff --git a/docs/conf.py b/docs/conf.py index 0a0c878..bade3de 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ def __getattr__(cls, name): # General information about the project. project = u'gamification-engine' -copyright = u'2015, Marcel Sander, Jens Janiuk' +copyright = u'2015, ActiDoo GmbH' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -204,7 +204,7 @@ def __getattr__(cls, name): # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'gamification-engine.tex', u'gamification-engine Documentation', - u'Marcel Sander, Jens Janiuk', 'manual'), + u'ActiDoo GmbH', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -234,7 +234,7 @@ def __getattr__(cls, name): # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'gamification-engine', u'gamification-engine Documentation', - [u'Marcel Sander, Jens Janiuk'], 1) + [u'ActiDoo GmbH'], 1) ] # If true, show URL addresses after external links. @@ -248,7 +248,7 @@ def __getattr__(cls, name): # dir menu entry, description, category) texinfo_documents = [ ('index', 'gamification-engine', u'gamification-engine Documentation', - u'Marcel Sander, Jens Janiuk', 'gamification-engine', 'One line description of project.', + u'ActiDoo GmbH', 'gamification-engine', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/internals/index.rst b/docs/internals/index.rst index be44cca..9666fb9 100644 --- a/docs/internals/index.rst +++ b/docs/internals/index.rst @@ -4,18 +4,18 @@ Modules ------- -.. image:: /_static/2015-09-07-erm.svg +.. image:: /_static/2017-03-28-erm.svg :width: 1000 -gengine.models +gengine.app.models ============== -.. automodule:: gengine.models - :members: ABase, User, Variable, Value, Achievement, Property, AchievementProperty, Reward, Goalproperty, GoalGoalproperty AchievementReward, AchievementUser, GoalEvaluationCache, Goal, Language, TranslationVariable, Translation +.. automodule:: gengine.app.models + :members: ABase, User, Variable, Value, Achievement, Property, AchievementProperty, Reward, GoalProperty, GoalGoalProperty, GoalTrigger, GoalTriggerStep, UserMessage, AchievementReward, AchievementUser, GoalEvaluationCache, Goal, Language, TranslationVariable, Translation, AuthUser, AuthToken, AuthRole, AuthRolePermission, UserDevice -gengine.views +gengine.app.views ============= -.. automodule:: gengine.views - :members: add_or_update_user, delete_user, get_progress, increas_value, get_achievement_level +.. automodule:: gengine.app.views + :members: add_or_update_user, delete_user, get_progress, increase_value, increase_multi_values, get_achievement_level, auth_login, register_device, get_messages, set_messages_read diff --git a/docs/rest/index.rst b/docs/rest/index.rst index f3bd12e..9d81c5e 100644 --- a/docs/rest/index.rst +++ b/docs/rest/index.rst @@ -18,6 +18,8 @@ Add or update user data - region (String city) - friends (comma separated list of user Ids) - groups (comma separated list of group Ids) + - language (name) + - additional_public_data (JSON) - add or updates a user with Id {userId} and Post parameters into the engines database - if friends Ids are not registered a empty record with only the user Id will be created @@ -32,10 +34,11 @@ Delete a user Increase Value ============== - - POST to "/increase_value/{variable_name}/{userId}" + - POST to "/increase_value/{variable_name}/{userId}/{key}" - URL parameters: - variable_name (the name of the variable to increase or decrease) - userId (the Id of the user) + - key (an optional key, describing the context of the event, can be used in rules) - POST parameters: - value (the increase/decrease value in Double) @@ -75,9 +78,9 @@ Get Progress Get a single achievement Level ============================== - - GET to "/increase_value/{variable_name}/{userId}" + - GET to "/achievement/{achievement_id}/level/{level}" - - can be used to check if a user is allowed to use a reward + - retrieves information about the rewards/properties of an achievement level Authentication ============================== diff --git a/docs/roadmap.rst b/docs/roadmap.rst index 2ef22d8..6f4ea5a 100644 --- a/docs/roadmap.rst +++ b/docs/roadmap.rst @@ -9,21 +9,17 @@ Features which might influence the overall performance or cause greater changes At ActiDoo.com we implement new functions as we need them and push them as soon as they are somewhat stable. -Important Todo +Todo ============== - - Tests (!) - -Less Important Todo -=================== + - Review and improve tests - Improve Caching Future Features =============== - possibility to store events (values table) in noSQL systems - - evaluate time-related leaderboards and reset them afterwards + - implement callback for time-aware achievements - nicer admin UI - statistics - maybe a possiblity to plugin authentication/authorization to allow users to directly push events to the engine - - this still needs to be discussed from an architectural point of view - - this would also introduce the need for security constraints to detect cheaters \ No newline at end of file + - this would also introduce the need for security constraints to detect cheaters \ No newline at end of file diff --git a/docs/upgrading/index.rst b/docs/upgrading/index.rst new file mode 100644 index 0000000..cd79867 --- /dev/null +++ b/docs/upgrading/index.rst @@ -0,0 +1,18 @@ +:title: gamification-engine installation +:description: installing the gamification-engine + +Upgrading +------------ + +From 0.1 to 0.2 +============ + +In version 0.2 we have introduced **breaking changes** that make it impossible to do an automatic upgrade. If you are happy with 0.1, there is no need to upgrade. +For performing a manual upgrade the following steps are required: + + - Install a new instance of 0.2 + - Recreate all settings / achievements manually using the new goal condition syntax + - Recreate users + - Copy values data + +*For future updates we will try to keep the goal condition syntax backwards compatible.* diff --git a/gengine/app/formular.py b/gengine/app/formular.py index 72a1ddf..627c83d 100644 --- a/gengine/app/formular.py +++ b/gengine/app/formular.py @@ -243,26 +243,26 @@ def evaluate_condition(inst, column_variable=None, column_key=None): demo_schema = { 'term': { 'variable': 'participate', - 'key_operator' : 'IN', + 'key_operator': 'IN', 'key': ['2', ], - 'type' : 'literal' + 'type': 'literal' } } demo2_schema = { 'term': { - 'type' : 'disjunction', + 'type': 'disjunction', 'terms': [ { - 'type' : 'literal', + 'type': 'literal', 'variable': 'participate', - 'key_operator' : 'ILIKE', + 'key_operator': 'ILIKE', 'key': ['%blah%', ] }, { - 'type' : 'literal', + 'type': 'literal', 'variable': 'participate', - 'key_operator' : 'IN', + 'key_operator': 'IN', 'key': ['2', ] } ] diff --git a/gengine/app/views.py b/gengine/app/views.py index 2aa25c4..8338baf 100644 --- a/gengine/app/views.py +++ b/gengine/app/views.py @@ -349,7 +349,7 @@ def get_achievement_level(request): if not achievement: raise APIError(404, "achievement_not_found", "achievement not found") - level_output = Achievement.basic_output(achievement, [], True, level).get("levels").get(str(level), {"properties":{},"rewards":{}}) + level_output = Achievement.basic_output(achievement, [], True, level).get("levels").get(str(level), {"properties": {}, "rewards": {}}) if "goals" in level_output: del level_output["goals"] if "level" in level_output: diff --git a/setup.py b/setup.py index 029f418..4a97678 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ "Topic :: Internet :: WWW/HTTP", "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", "Topic :: Software Development :: Libraries", - "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.5", "License :: OSI Approved :: MIT License" ], author='Marcel Sander, Jens Janiuk', From b559c93a09992d308cfea61e38feaf031a483d35 Mon Sep 17 00:00:00 2001 From: Sander <-> Date: Tue, 28 Mar 2017 11:15:24 +0200 Subject: [PATCH 167/176] 0.2 documentation --- docs/index.rst | 1 + docs/internals/index.rst | 13 ------------- docs/rest/index.rst | 17 +++++++++-------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 7d8db61..7cdae75 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,6 +20,7 @@ Contents: :maxdepth: 2 installing/index + upgrading/index concepts/index rest/index internals/index diff --git a/docs/internals/index.rst b/docs/internals/index.rst index 9666fb9..f95eac3 100644 --- a/docs/internals/index.rst +++ b/docs/internals/index.rst @@ -6,16 +6,3 @@ Modules .. image:: /_static/2017-03-28-erm.svg :width: 1000 - -gengine.app.models -============== - -.. automodule:: gengine.app.models - :members: ABase, User, Variable, Value, Achievement, Property, AchievementProperty, Reward, GoalProperty, GoalGoalProperty, GoalTrigger, GoalTriggerStep, UserMessage, AchievementReward, AchievementUser, GoalEvaluationCache, Goal, Language, TranslationVariable, Translation, AuthUser, AuthToken, AuthRole, AuthRolePermission, UserDevice - - -gengine.app.views -============= - -.. automodule:: gengine.app.views - :members: add_or_update_user, delete_user, get_progress, increase_value, increase_multi_values, get_achievement_level, auth_login, register_device, get_messages, set_messages_read diff --git a/docs/rest/index.rst b/docs/rest/index.rst index 9d81c5e..7fa487a 100644 --- a/docs/rest/index.rst +++ b/docs/rest/index.rst @@ -52,15 +52,16 @@ Increase multiple Values at once - POST to "/increase_multi_values" - JSON request body: .. code:: json + { - "{userId}" : { - "{variable}" : [ - { - "key" : "{key}", - "value" : "{value}" - } - ] - } + "{userId}" : { + "{variable}" : [ + { + "key" : "{key}", + "value" : "{value}" + } + ] + } } - directly evaluates all goals associated with the given variables From f5b516a0204f45dca6965cc8b3fbd186fe3cc74f Mon Sep 17 00:00:00 2001 From: Sander <-> Date: Tue, 28 Mar 2017 11:19:33 +0200 Subject: [PATCH 168/176] remove redundant evaluation --- docs/concepts/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/concepts/index.rst b/docs/concepts/index.rst index c13bf82..9488735 100644 --- a/docs/concepts/index.rst +++ b/docs/concepts/index.rst @@ -45,7 +45,6 @@ Goals define conditions that need to be fulfilled in order to get an achievement - group_by_key: group by the key of the values table - timespan: number of days which are considered (uses utc, i.e. days*24hours) - maxmin: "max" or "min" - select min or max value after grouping - - evaluation: "daily", "weekly", "monthly", "yearly" evaluation (users timezone) .. _PostgreSQL-Docs: http://www.postgresql.org/docs/9.3/static/functions-formatting.html From 0ca85b42023169e1a18ef8c33727177a95579b5b Mon Sep 17 00:00:00 2001 From: Sander <-> Date: Tue, 28 Mar 2017 11:20:52 +0200 Subject: [PATCH 169/176] remove python 2.x support --- .travis.yml | 1 - CHANGES.txt | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4f5350e..910f80d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python dist: trusty python: - - "2.7" - "3.4" - "3.5" sudo: required diff --git a/CHANGES.txt b/CHANGES.txt index 1348bd0..a5a72ec 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,3 +6,4 @@ * Introduce Mobile Pushes * Introduce Messages * Lots of bugfixes + * Remove Python 2.x support From 8f2dad9c8800505a3482b32e5a48824e30be379c Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Wed, 29 Mar 2017 10:53:27 +0200 Subject: [PATCH 170/176] Update documentation for version 0.2 --- docs/concepts/index.rst | 25 +++++++++++++++++++------ docs/installing/index.rst | 2 +- docs/rest/index.rst | 27 +++++++++++++-------------- docs/upgrading/index.rst | 2 +- 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/docs/concepts/index.rst b/docs/concepts/index.rst index 9488735..94e609d 100644 --- a/docs/concepts/index.rst +++ b/docs/concepts/index.rst @@ -4,7 +4,7 @@ Concepts -------- -Assumption: You installed the gamification-engine and can open the admin interface at /admin/ +Assumption: You installed the gamification-engine and you can open the admin interface at /admin/ Users ===== @@ -30,7 +30,7 @@ When such an event occurs, your application triggers the gamification engine to The storage of these values can be grouped by day, month or year to save storage. Note that if you want to specify time-based rules like "event X occurs Y times in the last 14 days", you may not group the values by month or year. -In addition to integers, the application can also set keys to model application-specific data. +In addition to integers, the application can also add additional keys to the variables to model application-specific data. Goals ===== @@ -50,7 +50,7 @@ Goals define conditions that need to be fulfilled in order to get an achievement The conditions contain a python expression that must evaluate to a valid parameter for SQLAlchemy's where function. -Examples: +### Examples: When the user has participated in the seminars 5, 7, and 9, he should get an achievement. We first need to create a variable "participate" and tell our application to increase the value of that variable with the seminar ID as key for the user by 1. @@ -101,7 +101,9 @@ For the first level, the user needs to invite 5 other users, for the second leve 5*level # level is set by the gamification engine -For further information about the rule language, we currently need to refer to .. _the sources: https://github.com/ActiDoo/gamification-engine/blob/develop/gengine/app/formular.py . +For further information about the rule language, we currently need to refer to the sources_ . + +.. _sources: https://github.com/ActiDoo/gamification-engine/blob/develop/gengine/app/formular.py Achievements ============ @@ -125,11 +127,11 @@ There is a *view_permission* setting that can be used when authorization is acti Properties ========== -A property describes Achievements or Goals of our system, like the name, image, description or XP the user should get. +A property describes an Achievement or a Goal of our system, like the name, image, description or XP the user should get. The Values of Properties can again be python formulas. Inside the formula you can make use of the level by using *level*. -Additionally Properties can be used as Variables. +Additionally, Properties can be used as Variables. This is useful to model goals like "reach 1000xp". @@ -142,3 +144,14 @@ the evaluated formulas of the rewards are compared to lower levels. The engine thus knows for each achieved level, which reward is new and can tell the application about this. In your application this could for example trigger a badge notification. + + +Further new concepts +======= +Since the latest version, some complete new optional concepts and features are added to the gamification-engine: + + - Authentication + - Push Notifications + - Messages + +All of these features are optional and they are not required to successfully use the engine. For the moment we refer to the source code and the description of the Rest API, a detailed documentation will follow. diff --git a/docs/installing/index.rst b/docs/installing/index.rst index d3f7bca..ce06381 100644 --- a/docs/installing/index.rst +++ b/docs/installing/index.rst @@ -7,7 +7,7 @@ Installation Requirements ============ -The gamification-engine requires an installed python distribution in version 2.7. It uses different language structures which are not supported in Python 2.6 or Python 3.x. Furthermore, the only currently supported persistence layer is PostgreSQL. +The gamification-engine requires an installed python distribution in version 3.x. It uses several language structures which are not supported in Python 2.x. Furthermore, the only currently supported persistence layer is PostgreSQL. Also the corresponding development packages are required (for Ubuntu/Debian: libpq-dev and python3-dev). Installation from PyPI diff --git a/docs/rest/index.rst b/docs/rest/index.rst index 7fa487a..7850600 100644 --- a/docs/rest/index.rst +++ b/docs/rest/index.rst @@ -39,8 +39,8 @@ Increase Value - variable_name (the name of the variable to increase or decrease) - userId (the Id of the user) - key (an optional key, describing the context of the event, can be used in rules) - - POST parameters: - - value (the increase/decrease value in Double) + - POST parameters: + - value (the increase/decrease value in Double) - if the userId is not registered an error will be thrown - directly evaluates all goals associated with this variable_name @@ -51,18 +51,17 @@ Increase multiple Values at once - POST to "/increase_multi_values" - JSON request body: - .. code:: json - - { - "{userId}" : { - "{variable}" : [ - { - "key" : "{key}", - "value" : "{value}" - } - ] - } - } + .. code:: json + { + "{userId}" : { + "{variable}" : [ + { + "key" : "{key}", + "value" : "{value}" + } + ] + } + } - directly evaluates all goals associated with the given variables - directly returns new reached achievements diff --git a/docs/upgrading/index.rst b/docs/upgrading/index.rst index cd79867..fbc639f 100644 --- a/docs/upgrading/index.rst +++ b/docs/upgrading/index.rst @@ -7,7 +7,7 @@ Upgrading From 0.1 to 0.2 ============ -In version 0.2 we have introduced **breaking changes** that make it impossible to do an automatic upgrade. If you are happy with 0.1, there is no need to upgrade. +In version 0.2 we have introduced **breaking changes** that make it impossible to do an automatic upgrade. If you are happy with 0.1, there is no need to upgrade. Furthermore, we have switched to Python 3.x as our main target environment. For performing a manual upgrade the following steps are required: - Install a new instance of 0.2 From 1adf6b80515e8be0eaf09cb5964a8dbbf5bdf605 Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Wed, 29 Mar 2017 10:59:05 +0200 Subject: [PATCH 171/176] Fix syntax error in docs --- docs/rest/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rest/index.rst b/docs/rest/index.rst index 7850600..9e8608b 100644 --- a/docs/rest/index.rst +++ b/docs/rest/index.rst @@ -39,7 +39,7 @@ Increase Value - variable_name (the name of the variable to increase or decrease) - userId (the Id of the user) - key (an optional key, describing the context of the event, can be used in rules) - - POST parameters: + - POST parameters: - value (the increase/decrease value in Double) - if the userId is not registered an error will be thrown From 3be4e92555cff1047222c329095672e558c277b7 Mon Sep 17 00:00:00 2001 From: Sander <-> Date: Wed, 29 Mar 2017 12:21:22 +0200 Subject: [PATCH 172/176] separate optional dependencies --- optional-requirements.txt | 11 ++++++++ requirements.txt | 57 +++++++++++++++++++-------------------- setup.py | 23 ++++++++++------ 3 files changed, 54 insertions(+), 37 deletions(-) create mode 100644 optional-requirements.txt diff --git a/optional-requirements.txt b/optional-requirements.txt new file mode 100644 index 0000000..39258ea --- /dev/null +++ b/optional-requirements.txt @@ -0,0 +1,11 @@ +argon2==0.1.10 +names==0.3.0 +pbr==2.0.0 +pg8000==1.10.6 +python-gcm==0.4 +redis==2.10.5 +requests==2.13.0 +tapns3==3.0.0 +testing.common.database==2.0.0 +testing.postgresql==1.3.0 +testing.redis==1.1.1 diff --git a/requirements.txt b/requirements.txt index 59edaa2..6e6912e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,45 +1,44 @@ -alembic==0.8.8 -tapns3==3.0.0 -argon2==0.1.10 -Chameleon==2.25 -click==6.6 +alembic==0.9.1 +appdirs==1.4.3 +Chameleon==3.1 +click==6.7 dogpile.cache==0.6.2 -Flask==0.11.1 -Flask-Admin==1.4.2 +Flask==0.12 +Flask-Admin==1.5.0 +hupper==0.4.4 itsdangerous==0.24 -Jinja2==2.8 +Jinja2==2.9.5 jsl==0.2.4 -jsonschema==2.5.1 -Mako==1.0.5 -MarkupSafe==0.23 +jsonschema==2.6.0 +Mako==1.0.6 +MarkupSafe==1.0 mock==2.0.0 +packaging==16.8 PasteDeploy==1.5.2 -pbr==1.10.0 -psycopg2==2.6.2 -Pygments==2.1.3 -pymemcache==1.4.0 -pyparsing==2.1.10 -pyramid==1.7.3 +pbr==2.0.0 +psycopg2==2.7.1 +Pygments==2.2.0 +pymemcache==1.4.2 +pyparsing==2.2.0 +pyramid==1.8.3 pyramid-chameleon==0.3 pyramid-debugtoolbar==3.0.5 pyramid-dogpile-cache==0.0.4 pyramid-mako==1.0.2 -pyramid-tm==1.0.1 -python-editor==1.0.1 -python-gcm==0.4 -pytz==2016.7 -raven==5.31.0 +pyramid-tm==1.1.1 +python-editor==1.0.3 +pytz==2016.10 +raven==6.0.0 repoze.lru==0.6 -requests==2.11.1 six==1.10.0 -SQLAlchemy==1.1.3 -transaction==1.6.1 +SQLAlchemy==1.1.7 +transaction==2.1.2 translationstring==1.3 venusian==1.0 -waitress==1.0.1 -WebOb==1.6.2 -Werkzeug==0.11.11 +waitress==1.0.2 +WebOb==1.7.2 +Werkzeug==0.12.1 WTForms==2.1 zope.deprecation==4.2.0 -zope.interface==4.3.2 +zope.interface==4.3.3 zope.sqlalchemy==0.7.7 diff --git a/setup.py b/setup.py index 4a97678..0a28feb 100644 --- a/setup.py +++ b/setup.py @@ -31,14 +31,7 @@ 'jsl', 'jsonschema', 'pyparsing', - 'argon2', - 'tapns3', - 'python-gcm', - #Testing: - 'testing.postgresql', - 'testing.redis', - 'names' - ] +] version = '' with open('gengine/__init__.py', 'r') as fd: @@ -71,6 +64,20 @@ zip_safe=False, test_suite='gengine', install_requires=requires, + extras_require={ + "auth": [ + 'argon2' + ], + "pushes": [ + 'tapns3', + 'python-gcm', + ], + "testing": [ + 'testing.postgresql', + 'testing.redis', + 'names' + ] + }, entry_points="""\ [paste.app_factory] main = gengine:main From d5d345354e043be77f56b20b38ec3bd820de2e9e Mon Sep 17 00:00:00 2001 From: Sander <-> Date: Wed, 29 Mar 2017 12:31:53 +0200 Subject: [PATCH 173/176] don't die when optional dependencies are missing --- gengine/app/push.py | 17 +++++++++--- gengine/app/tests/db.py | 9 ++++++- gengine/app/tests/helpers.py | 9 ++++++- gengine/app/tests/runner.py | 6 ++++- gengine/maintenance/scripts/initializedb.py | 29 ++++++++++++--------- 5 files changed, 51 insertions(+), 19 deletions(-) diff --git a/gengine/app/push.py b/gengine/app/push.py index f92b7d2..64ae5d4 100644 --- a/gengine/app/push.py +++ b/gengine/app/push.py @@ -1,8 +1,6 @@ -import logging import random import threading -from apns import APNs, Payload -from gcm import GCM + import os from sqlalchemy.sql.expression import and_, select from sqlalchemy.sql.functions import func @@ -13,8 +11,21 @@ from gengine.metadata import DBSession threadlocal = threading.local() + +import logging log = logging.getLogger(__name__) +try: + from apns import APNs, Payload +except ImportError as e: + log.info("tapns3 not installed") + +try: + from gcm import GCM +except ImportError as e: + log.info("python-gcm not installed") + + def get_prod_apns(): """ http://stackoverflow.com/questions/1762555/creating-pem-file-for-apns diff --git a/gengine/app/tests/db.py b/gengine/app/tests/db.py index ecdb98a..21eb8a1 100644 --- a/gengine/app/tests/db.py +++ b/gengine/app/tests/db.py @@ -1,6 +1,13 @@ -import testing.postgresql import os +import logging +log = logging.getLogger(__name__) + +try: + import testing.postgresql +except ImportError as e: + log.info("testing.postgresql not installed") + db = None def setupDB(): diff --git a/gengine/app/tests/helpers.py b/gengine/app/tests/helpers.py index b05a6bc..6ab5a51 100644 --- a/gengine/app/tests/helpers.py +++ b/gengine/app/tests/helpers.py @@ -1,4 +1,3 @@ -import names import random import datetime @@ -9,6 +8,14 @@ from gengine.app.model import UserDevice, t_user_device from sqlalchemy import and_, select +import logging +log = logging.getLogger(__name__) + +try: + import names +except ImportError as e: + log.info("names not installed") + default_gen_data = { "timezone" : "Europe/Berlin", "area" : { diff --git a/gengine/app/tests/runner.py b/gengine/app/tests/runner.py index 0a4307a..a25393c 100644 --- a/gengine/app/tests/runner.py +++ b/gengine/app/tests/runner.py @@ -3,12 +3,16 @@ import unittest import os import pkgutil -import testing.redis import logging import sys log = logging.getLogger(__name__) +try: + import testing.redis +except ImportError as e: + log.info("testing.redis not installed") + init_session() init_declarative_base() diff --git a/gengine/maintenance/scripts/initializedb.py b/gengine/maintenance/scripts/initializedb.py index 2c0a562..b79f944 100644 --- a/gengine/maintenance/scripts/initializedb.py +++ b/gengine/maintenance/scripts/initializedb.py @@ -290,19 +290,22 @@ def add_translation(variable, lang, text): DBSession.add(user3) DBSession.flush() - auth_user = AuthUser(user_id=user1.id,email="admin@gamification-software.com",password="test123",active=True) - DBSession.add(auth_user) - - auth_role = AuthRole(name="Global Admin") - DBSession.add(auth_role) - - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_access_admin_ui)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_delete_user)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_increase_value)) - DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_update_user_infos)) - - auth_user.roles.append(auth_role) - DBSession.add(auth_user) + try: + auth_user = AuthUser(user_id=user1.id,email="admin@gamification-software.com",password="test123",active=True) + DBSession.add(auth_user) + + auth_role = AuthRole(name="Global Admin") + DBSession.add(auth_role) + + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_access_admin_ui)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_delete_user)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_increase_value)) + DBSession.add(AuthRolePermission(role=auth_role, name=perm_global_update_user_infos)) + + auth_user.roles.append(auth_role) + DBSession.add(auth_user) + except ImportError as e: + print("[auth] feature not installed - not importing auth demo data") if __name__ == '__main__': From dbbe5411e8194dacbec5feaf4b91701dc4e5d26d Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Fri, 7 Apr 2017 14:15:50 +0200 Subject: [PATCH 174/176] Explain extras dependencies --- docs/installing/index.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/installing/index.rst b/docs/installing/index.rst index ce06381..952fc34 100644 --- a/docs/installing/index.rst +++ b/docs/installing/index.rst @@ -24,6 +24,14 @@ You can install it by invoking $ gengine_quickstart mygengine $ cd mygengine +In the latest version, there are some optional dependencies for auth pushes and testing. To use these features install it in the following way: + +.. highlight:: bash + +:: + + $ pip install gamification-engine[auth,pushes,testing] + Afterwards edit production.ini according to your needs. Database From 2229844f47e0640a10bad99798eb73a6e9bdef0c Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Fri, 7 Apr 2017 14:41:09 +0200 Subject: [PATCH 175/176] Prepare new version --- CHANGES.txt | 8 ++++---- debug.py | 0 setup.py | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 debug.py diff --git a/CHANGES.txt b/CHANGES.txt index a5a72ec..6010065 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,9 +1,9 @@ -0.2 +0.2.0 * Implement new rule syntax * Add time-aware / recurring achievements * Add optional authentication & authorization - * Introduce Goal Triggers - * Introduce Mobile Pushes - * Introduce Messages + * Introduce goal triggers + * Introduce mobile pushes + * Introduce messages * Lots of bugfixes * Remove Python 2.x support diff --git a/debug.py b/debug.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index 0a28feb..96bdbdc 100644 --- a/setup.py +++ b/setup.py @@ -51,10 +51,11 @@ "Topic :: Internet :: WWW/HTTP", "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", "Topic :: Software Development :: Libraries", + "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "License :: OSI Approved :: MIT License" ], - author='Marcel Sander, Jens Janiuk', + author='Marcel Sander, Jens Janiuk, Matthias Feldotto', author_email='marcel@gamification-software.com', license='MIT', url='https://www.gamification-software.com', @@ -90,7 +91,4 @@ [redgalaxy.plugins] gengine = gengine:redgalaxy """, - dependency_links=[ - "https://github.com/ActiDoo/PyAPNs/tarball/master#egg=apns", - ], ) From 55f23ce73b59ae2111eb36a3a095d894c5c56d6f Mon Sep 17 00:00:00 2001 From: Matthias Feldotto Date: Fri, 7 Apr 2017 14:44:12 +0200 Subject: [PATCH 176/176] Install optional requirements for travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 910f80d..b136e5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ apt: install: - pip install coveralls - "pip install --upgrade -r requirements.txt" + - "pip install --upgrade -r optional-requirements.txt" - pip install -e . # command to run tests