diff --git a/.gitignore b/.gitignore index 292fd04..38a4b7d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ darma-task # for PyCharm Users .idea *.log +boteval/templates diff --git a/boteval/app.py b/boteval/app.py index 3b5ae4b..bf7711f 100644 --- a/boteval/app.py +++ b/boteval/app.py @@ -74,6 +74,9 @@ def init_app(**args): app.config['SQLALCHEMY_DATABASE_URI'] = db_uri log.info(f'SQLALCHEMY_DATABASE_URI = {db_uri}') + scheme = app.config.get('PREFERRED_URL_SCHEME', 'http') + app.config['PREFERRED_URL_SCHEME'] = scheme + if (task_dir / '__init__.py').exists(): # task dir has python code log.info(f'{task_dir} is a python module. Going to import it.') load_dir_as_module(task_dir) @@ -118,7 +121,10 @@ def main(): base_prefix = args.get('base') or '/' log.info(f'Internal URL http://{host}:{port}{base_prefix}') #socket.run(app, port=port, host=host, debug=app.debug) - app.run(port=port, host=host) + if app.config['PREFERRED_URL_SCHEME'] == 'https': + app.run(port=port, host=host, ssl_context='adhoc') + else: + app.run(port=port, host=host) if __name__ == "__main__": main() diff --git a/boteval/bots.py b/boteval/bots.py index 834b916..61349a8 100644 --- a/boteval/bots.py +++ b/boteval/bots.py @@ -25,18 +25,18 @@ def __init__(self, *args, name=None, **kwargs) -> None: self.update_signature(agent_name=name) self.last_msg = None - def get_name(self) -> str: - raise NotImplementedError() - - def update_signature(self, **kwargs): - self.signature.update(kwargs) + def init_chat_context(self, init_messages: List[Dict[str, Any]]): + raise NotImplementedError(f'{type(self)} must implement init_chat_context() method') def hear(self, msg: Dict[str, Any]): - self.last_msg = msg + raise NotImplementedError(f'{type(self)} must implement hear() method') def talk(self) -> Dict[str, Any]: raise NotImplementedError(f'{type(self)} must implement talk() method') + def update_signature(self, **kwargs): + self.signature.update(kwargs) + def interactive_shell(self): log.info(f'Launching an interactive shell with {type(self)}.\n' 'Type "exit" or press CTRL+D to quit.') @@ -60,11 +60,13 @@ class DummyBot(BotAgent): NAME = 'dummybot' - def talk(self): + def init_chat_context(self, init_messages: List[Dict[str, Any]]): + log.debug(f'Initializing chat context with {len(init_messages)} messages') + def talk(self, **kwargs): context = (self.last_msg or {}).get('text', '') if context.lower() == 'ping': return 'pong' - return dict(text="dummybot reply --" + context[-30:]) + return dict(text="dummybot reply --" + context[-30:], data={'speaker_id': 'Moderator'}) @R.register(R.BOT, 'hf-transformers') diff --git a/boteval/constants.py b/boteval/constants.py index 87a7e25..1e69748 100644 --- a/boteval/constants.py +++ b/boteval/constants.py @@ -51,3 +51,5 @@ class Auth: AWS_MAX_RESULTS = 100 MTURK_LOG_LEVEL = logging.INFO +BOT_REPLY_DELAY_START = 5.0 # bot reply delay in seconds, value is small for debugging +BOT_REPLY_DELAY_END = 10.0 # bot reply delay in seconds diff --git a/boteval/controller.py b/boteval/controller.py index 3bad0ad..8eb5900 100644 --- a/boteval/controller.py +++ b/boteval/controller.py @@ -1,4 +1,3 @@ - import functools from typing import List, Tuple import random @@ -12,7 +11,6 @@ from boteval.service import ChatService - from . import log, C, db from .utils import jsonify, render_template, register_template_filters from .model import ChatMessage, ChatThread, ChatTopic, User, SuperTopic @@ -25,13 +23,12 @@ def wrap(body=None, status=C.SUCCESS, description=None): body=body) - class AdminLoginDecorator: - """ Similar to flask-login's `login_required` but checks for role=admin on user This is a stateful decorator (hence class instead of function) """ + def __init__(self, login_manager=None) -> None: self.login_manager = login_manager @@ -44,18 +41,18 @@ def decorated_view(*args, **kwargs): elif FL.current_user.role != User.ROLE_ADMIN: flask.flash('This functionality is for admin only') return 'ERROR: Access denied. This resource can only be accessed by admin user.', 403 - else: # user is logged in and they have role=admin; + else: # user is logged in and they have role=admin; return func(*args, **kwargs) return decorated_view + admin_login_required = None def init_login_manager(login_manager): - global admin_login_required - admin_login_required = AdminLoginDecorator(login_manager=login_manager) + admin_login_required = AdminLoginDecorator(login_manager=login_manager) @login_manager.user_loader def load_user(user_id: str): @@ -71,18 +68,18 @@ def is_safe_url(url): # TODO: validate url return True -def register_app_hooks(app, service: ChatService): +def register_app_hooks(app, service: ChatService): @app.before_request def update_last_active(): - user:User = FL.current_user + user: User = FL.current_user if user and user.is_active: - if not user.last_active or\ - (datetime.now() - user.last_active).total_seconds() > C.USER_ACTIVE_UPDATE_FREQ: + if not user.last_active or \ + (datetime.now() - user.last_active).total_seconds() > C.USER_ACTIVE_UPDATE_FREQ: user.last_active = datetime.now() db.session.merge(user) # update db.session.commit() - + @app.before_first_request def before_first_request(): log.info('Before first request') @@ -91,7 +88,6 @@ def before_first_request(): def user_controllers(router, service: ChatService): - @router.route('/ping', methods=['GET', 'POST']) def ping(): return jsonify(dict(reply='pong', time=datetime.now().timestamp())), 200 @@ -140,7 +136,8 @@ def login(): if user: flask.flash(f'User {user.id} already exists. Try login instead.') elif len(user_id) < 2 or len(user_id) > 16 or not user_id.isalnum(): - flask.flash('Invalid User ID. ID should be at least 2 chars and atmost 16 chars and only alpha numeric chars are permitted') + flask.flash( + 'Invalid User ID. ID should be at least 2 chars and atmost 16 chars and only alpha numeric chars are permitted') elif len(secret) < 4: flask.flash('Password should be atleast 4 chars long') else: @@ -149,14 +146,14 @@ def login(): ext_src = args.pop('ext_src', None) user = User.create_new(user_id, secret, name=name, ext_id=ext_id, ext_src=ext_src, data=args) tmpl_args['action'] = 'login' - flask.flash(f'Sign up success. Try login with your user ID: {user.id}. Verify that it works and write down the password for future logins.') + flask.flash( + f'Sign up success. Try login with your user ID: {user.id}. Verify that it works and write down the password for future logins.') return render_template('login.html', **tmpl_args) else: flask.flash('Wrong action. only login and signup are supported') tmpl_args['action'] = 'login' return render_template('login.html', user_id=user_id, **tmpl_args) - @router.route('/seamlesslogin', methods=['GET', 'POST']) def seamlesslogin(): next_url = request.values.get('next') @@ -175,7 +172,7 @@ def seamlesslogin(): ext_src=ext_src, onboarding=service.onboarding) log.info(f"login/signup. next={next_url} | ext: src: {ext_src} id: {ext_id}") - if request.method == 'GET': # for GET, show terms, + if request.method == 'GET': # for GET, show terms, return render_template('seamlesslogin.html', **tmpl_args) # form sumission as POST => create a/c @@ -184,7 +181,7 @@ def seamlesslogin(): secret = args.pop('secret') user = User.get(user_id) log.info(f'Form:: {user_id} {args}') - if user:# user already exists + if user: # user already exists log.warning(f'User {user_id} already exists') else: name = args.pop('name', None) @@ -196,7 +193,6 @@ def seamlesslogin(): return flask.redirect(next_url) return flask.redirect(flask.url_for('app.index')) - @router.route('/logout', methods=['GET']) @FL.login_required def logout(): @@ -207,12 +203,11 @@ def logout(): @router.route('/about', methods=['GET']) def about(): return render_template('about.html') - + @router.route('/instructions', methods=['GET']) def instructions(focus_mode=False): return render_template('page.html', content=service.instructions, focus_mode=focus_mode) - @router.route('/', methods=['GET']) @FL.login_required def index(): @@ -224,7 +219,7 @@ def index(): threads_completed=sum(th.episode_done for th in threads)) limits.update(service.limits) max_threads_per_topic = limits['max_threads_per_topic'] - thread_counts = service.get_thread_counts(episode_done=True) # completed threads + thread_counts = service.get_thread_counts(episode_done=True) # completed threads for thread in threads: if ChatTopic.query.get(thread.topic_id) is None: continue @@ -234,13 +229,14 @@ def index(): data[topic_id][2] = n_threads data = list(data.values()) random.shuffle(data) # randomize + # Ranks: threads t threads that need anotation in the top def sort_key(rec): _, my_thread, n_threads = rec if my_thread: - if not my_thread.episode_done: # Rank 1 incomplete threads, + if not my_thread.episode_done: # Rank 1 incomplete threads, return -1 - else: # completed thread + else: # completed thread return max_threads_per_topic # done, push it to the end else: return n_threads @@ -248,7 +244,6 @@ def sort_key(rec): data = list(sorted(data, key=sort_key)) return render_template('user/index.html', data=data, limits=limits) - @router.route('/launch-topic/', methods=['GET']) @FL.login_required def launch_topic(topic_id): @@ -299,53 +294,38 @@ def get_thread(thread_id, request_worker_id=None, focus_mode=False): req_worker_is_human_mod = False instructions_for_user = service.instructions - if topic.human_moderator == 'yes' and request_worker_id is not None and request_worker_id != '': - req_worker_is_human_mod = service.crowd_service.is_worker_qualified(user_worker_id=request_worker_id, - qual_name='human_moderator_qualification') - - if req_worker_is_human_mod: - instructions_for_user = service.human_mod_instructions - # add one more turn for human mod: - remaining_turns = remaining_turns + 1 - log.info('Human moderator instructions should be used for user:', request_worker_id) - - if thread.max_human_users_per_thread == 1: - return render_template('user/chatui.html', limits=service.limits, - thread_json=json.dumps(thread.as_dict(), ensure_ascii=False), - thread=thread, - topic=topic, - socket_name=thread.socket_name, - rating_questions=ratings, - focus_mode=focus_mode, - remaining_turns=remaining_turns, - instructions_html=service.instructions, - simple_instructions_html=service.simple_instructions, - show_text_extra=FL.current_user.is_admin, - data=dict()) - elif thread.max_human_users_per_thread == 2: - return render_template('user/chatui_two_users.html', limits=service.limits, - thread_json=json.dumps(thread.as_dict(), ensure_ascii=False), - thread=thread, - topic=topic, - socket_name=thread.socket_name, - rating_questions=ratings, - focus_mode=focus_mode, - remaining_turns=remaining_turns, - instructions_html=instructions_for_user, - simple_instructions_html=service.simple_instructions, - show_text_extra=FL.current_user.is_admin, - bot_name=C.Auth.BOT_USER, - data=dict()) + # if topic.human_moderator == 'yes' and request_worker_id is not None and request_worker_id != '': + # req_worker_is_human_mod = service.crowd_service.is_worker_qualified(user_worker_id=request_worker_id, + # qual_name='human_moderator_qualification') + # + # if req_worker_is_human_mod: + # instructions_for_user = service.human_mod_instructions + # # add one more turn for human mod: + # remaining_turns = remaining_turns + 1 + # log.info('Human moderator instructions should be used for user:', request_worker_id) + + return render_template('user/chatui/chatui.html', limits=service.limits, + thread_json=json.dumps(thread.as_dict(), ensure_ascii=False), + thread=thread, + topic=topic, + socket_name=thread.socket_name, + rating_questions=ratings, + focus_mode=focus_mode, + remaining_turns=remaining_turns, + instructions_html=service.instructions, + simple_instructions_html=service.simple_instructions, + show_text_extra=FL.current_user.is_admin, + data=dict()) @router.route('/thread///message', methods=['POST']) - #@FL.login_required <-- login didnt work in iframe in mturk + # @FL.login_required <-- login didnt work in iframe in mturk def post_new_message(thread_id, user_id): thread = service.get_thread(thread_id) if not thread: return f'Thread {thread_id} NOT found', 404 - #user = FL.current_user + # user = FL.current_user user = User.get(user_id) - if not user or user not in thread.users: + if not user or user not in thread.users: log.warning('user is not part of thread') reply = dict(status=C.ERROR, description=f'User {user.id} is not part of thread {thread.id}. Wrong thread!') @@ -356,7 +336,7 @@ def post_new_message(thread_id, user_id): reply = dict(status=C.ERROR, description=f'requires "text" field of type string') return flask.jsonify(reply), 400 - + text = text[:C.MAX_TEXT_LENGTH] msg = ChatMessage(text=text, user_id=user.id, thread_id=thread.id, data={"speaker_id": speaker_id}) try: @@ -367,29 +347,27 @@ def post_new_message(thread_id, user_id): log.exception(e) return flask.jsonify(dict(status=C.ERROR, description='Something went wrong on server side')), 500 - - # same as above but just post current thread without new text + # same as above but just post current thread without new text @router.route('/thread///current_thread', methods=['POST']) - def post_current_thread(thread_id, user_id): + def post_current_thread(thread_id, user_id): thread = service.get_thread(thread_id) - if not thread: + if not thread: return f'Thread {thread_id} NOT found', 404 - + user = User.get(user_id) if not user or user not in thread.users: log.warning('user is not part of thread') reply = dict(status=C.ERROR, description=f'User {user.id} is not part of thread {thread.id}. Wrong thread!') return flask.jsonify(reply), 400 - - try: + + try: reply, episode_done = service.current_thread(thread) reply_dict = reply.as_dict() | dict(episode_done=episode_done) return flask.jsonify(reply_dict), 200 except Exception as e: log.exception(e) return flask.jsonify(dict(status=C.ERROR, description='Something went wrong on server side')), 500 - @router.route('/thread///latest_message', methods=['GET']) def get_latest_message(thread_id, user_id): @@ -423,10 +401,10 @@ def get_thread_object(thread_id) -> tuple[Response, int]: return flask.jsonify(thread.as_dict()), 200 @router.route('/thread///rating', methods=['POST']) - #@FL.login_required <-- login didnt work in iframe in mturk + # @FL.login_required <-- login didnt work in iframe in mturk def thread_rating(thread_id, user_id): thread = service.get_thread(thread_id) - user = User.get(user_id) # FL.current_user + user = User.get(user_id) # FL.current_user if not thread: return f'Thread {thread_id} NOT found', 404 if not user: @@ -448,60 +426,138 @@ def thread_rating(thread_id, user_id): flask.flash(note_text) return flask.redirect(url_for('app.index')) + @router.route('api/post-message', methods=['POST']) + def post_message(): + user_id = request.form.get('user_id', None) + thread_id = request.form.get('thread_id', None) + if not thread_id: + return f'Thread ID is required', 400 + + thread = ChatThread.get_thread_by_id(thread_id) + if not thread: + return f'Thread {thread_id} NOT found', 404 + + user = User.get(user_id) + msg = ChatMessage( + text=request.form.get('text', None), + user_id=user.id, + thread_id=thread_id, + data={"speaker_id": request.form.get('speaker_id', None)} + ) + success, info = thread.append_message(msg) + + if success: + msg.save_message_to_db() + thread.update_thread_in_db() + return jsonify(dict( + status=C.SUCCESS, + message_id=msg.id, + timestamp=msg.time_created, + )), 200 + else: + return jsonify(dict(status=C.ERROR, description=info)), 400 + + _avoid_double_submit = dict() + + @router.route('api/get-bot-reply', methods=['POST']) + def get_bot_reply(): + thread_id = request.form.get('thread_id', None) + if not thread_id: + return f'Thread ID is required', 400 + + thread = ChatThread.get_thread_by_id(thread_id) + + if not thread: + return f'Thread {thread_id} NOT found', 404 + + user_id = request.form.get('user_id', None) + post_turns = int(request.form.get('turns', 0)) + post_idx = int(request.form.get('speaker_idx', 0)) + idx = post_turns * len(thread.speak_order) + post_idx + if thread.id in _avoid_double_submit and _avoid_double_submit[thread.id] >= idx: + reply = dict(status=C.ERROR, + description=f'Bot reply idx{idx} already create. Ignoring this request.') + return flask.jsonify(reply), 400 + + user = User.get(user_id) + if not user or user not in thread.users: + log.warning('user is not part of thread') + reply = dict(status=C.ERROR, + description=f'User {user.id} is not part of thread {thread.id}. Wrong thread!') + return flask.jsonify(reply), 400 + + try: + msg = service.get_dialog_man(thread).bot_reply() + msg.save_message_to_db() + thread.append_message(msg) + thread.update_thread_in_db() + reply_dict = msg.as_dict() + return flask.jsonify(reply_dict), 200 + except Exception as e: + log.exception(e) + return flask.jsonify(dict(status=C.ERROR, description='Something went wrong on server side')), 500 + ########### M Turk Integration ######################### @router.route('/mturk-landing/', methods=['GET']) def mturk_landing(topic_id): # this is where mturk worker should land first - assignment_id = request.values.get('assignmentId') - is_previewing = assignment_id == 'ASSIGNMENT_ID_NOT_AVAILABLE' + if assignment_id == 'ASSIGNMENT_ID_NOT_AVAILABLE': + # In preview mode: show instructions + return instructions(focus_mode=True) + hit_id = request.values.get('hitId') - worker_id = request.values.get('workerId') # wont be available while previewing - submit_url = request.values.get('turkSubmitTo', '') # wont be available while previewing + worker_id = request.values.get('workerId') # wont be available while previewing + submit_url = request.values.get('turkSubmitTo', '') # wont be available while previewing if not hit_id: return f'HITId not found. {assignment_id=} {worker_id=} This URL is reserved for Mturk users only: {submit_url}', 400 - - # because Jon suggested we make it seamless for mturk users - seamless_login = service.config.is_seamless_crowd_login - ext_src = C.MTURK_SANDBOX if 'sandbox' in submit_url else C.MTURK + + ext_src = C.MTURK_SANDBOX if 'sandbox' in submit_url else C.MTURK if submit_url: submit_url = submit_url.rstrip('/') + '/mturk/externalSubmit' - - #Our mapping: Worker=User; Assignment = ChatThread; HIT=ChatTopic - # Step 1: map Hit to Topic, so we can perview it - topic = ChatTopic.query.filter_by(ext_id=hit_id).first() - if not topic: - return 'Invalid HIT or topic ID.', 400 - # We shouldn't check limit here, because we don't know the user yet. - # If the user has already participated in this task, we should still allow them to review the history. - # So, the following code is commented out. - # limit_exceeded, msg = service.limit_check(topic=topic, user=None) - # if limit_exceeded: - # return msg, 400 - - if is_previewing: - return instructions(focus_mode=True) # sending index page for now. We can do better later + # Step 1: map Hit to Topic, so we can perview it + # topic = ChatTopic.query.filter_by(ext_id=hit_id).first() + # if not topic: + # return 'Invalid HIT or topic ID.', 400 # Step2. Find the mapping user user = User.query.filter_by(ext_src=ext_src, ext_id=worker_id).first() - if not user: # sign up and come back (so set next) - + if not user: # sign up and come back (so set next) + return flask.redirect( url_for('app.seamlesslogin', ext_id=worker_id, ext_src=ext_src, next=request.url)) - + if not FL.current_user or FL.current_user.get_id() != user.id: FL.logout_user() FL.login_user(user, remember=True, force=True) - - limit_exceeded, msg = service.limit_check(topic=topic, user=user) - if limit_exceeded: # we may need to block user i.e. change user qualification - return msg, 400 + + # limit_exceeded, msg = service.limit_check(topic=topic, user=user) + # if limit_exceeded: # we may need to block user i.e. change user qualification + # return msg, 400 + + def _get_thread(): + topics = ChatTopic.get_topics_user_not_done(user) + if len(topics) == 0: + return None + # Check if someone waiting in an existing thread + for topic in topics: + thread = ChatThread.get_next_vacancy_thread_for_topic(topic) + if thread is not None: + thread = service.get_thread_for_topic(user=FL.current_user, topic=topic, create_if_missing=True) + return thread + + thread = service.get_thread_for_topic(user=FL.current_user, topic=topics[0], create_if_missing=True) + return thread + + thread = _get_thread() + if not thread: + return 'No more threads left to annotate. Thank you for your contribution!', 200 # user exist, logged in => means user already signed up and doing so, gave consent, # Step 3: map assignment to chat thread -- data = { - ext_src : { + ext_src: { 'submit_url': submit_url, 'is_sandbox': 'workersandbox' in submit_url, 'assignment_id': assignment_id, @@ -509,18 +565,20 @@ def mturk_landing(topic_id): # this is where mturk worker should land first 'worker_id': worker_id } } - chat_thread = service.get_thread_for_topic(user=FL.current_user, topic=topic, create_if_missing=True, - ext_id=assignment_id, ext_src=ext_src, data=data) - - if chat_thread is None: - err_msg = 'Another user is loading this chat topic. Please retry after 10 seconds!' - log.error(err_msg) - return err_msg, 400 + thread.set_external_info_for_user(user, assignment_id, ext_src, data) + thread.update_thread_in_db() + # chat_thread = service.get_thread_for_topic(user=FL.current_user, topic=topic, create_if_missing=True, + # ext_id=assignment_id, ext_src=ext_src, data=data) + + # if chat_thread is None: + # err_msg = 'Another user is loading this chat topic. Please retry after 10 seconds!' + # log.error(err_msg) + # return err_msg, 400 - log.info(f'chat_thread: {chat_thread}') - log.info(f'worker_id: {worker_id}') - return get_thread(thread_id=chat_thread.id, request_worker_id=worker_id, focus_mode=True) + # log.info(f'chat_thread: {chat_thread}') + # log.info(f'worker_id: {worker_id}') + return get_thread(thread_id=thread.id, request_worker_id=worker_id, focus_mode=True) ###################### ADMIN STUFF ##################### @@ -592,7 +650,7 @@ def get_topics(): topic_thread_counts_dict = {topic: topic_thread_counts.get(topic.id, 0) for topic in all_topics} super_topics = \ [(super_topic, super_topic_thread_counts.get(super_topic.id, 0)) for super_topic in all_super_topics] - return render_template('admin/topics.html', tasks=all_topics, super_topics=super_topics, + return render_template('admin/topics/topics.html', tasks=all_topics, super_topics=super_topics, external_url_ok=service.is_external_url_ok, **admin_templ_args, topic_thread_counts_dict=topic_thread_counts_dict, service=service) else: @@ -613,7 +671,7 @@ def get_topics(): max_turns_per_thread=int(args['max_turns_per_thread']), max_human_users_per_thread=int(args['max_human_users_per_thread']), human_moderator=args['human_moderator'], - reward=args['reward']) + reward=args['reward'], parameters=args) elif "multi-tasks-launch" in args.keys(): selected_task_ids = request.form.getlist('multi-tasks-launch') for task_id in selected_task_ids: @@ -625,7 +683,7 @@ def get_topics(): max_turns_per_thread=int(args['max_turns_per_thread']), max_human_users_per_thread=int(args['max_human_users_per_thread']), human_moderator=args['human_moderator'], - reward=args['reward']) + reward=args['reward'], parameters=args) return redirect(url_for('admin.get_topics')) @router.route(f'/topic//launch/') diff --git a/boteval/model.py b/boteval/model.py index aa995f6..9263297 100644 --- a/boteval/model.py +++ b/boteval/model.py @@ -1,8 +1,7 @@ -from typing import Dict, List, Optional, Any +from typing import Dict, List, Optional, Any, Tuple import hashlib from sqlalchemy import orm, sql - from . import db, log """ @@ -20,6 +19,17 @@ """ +def save_to_db(obj): + db.session.add(obj) + db.session.flush() + db.session.commit() + + +def update_in_db(obj): + db.session.merge(obj) + db.session.flush() + db.session.commit() + class BaseModel(db.Model): __abstract__ = True @@ -49,37 +59,33 @@ def __hash__(self) -> int: def as_dict(self) -> Dict[str, Any]: return dict( - id = self.id, - data = self.data if self.data is not None else dict(), + id=self.id, + data=self.data if self.data is not None else dict(), time_created=self.time_created and self.time_created.isoformat(), time_updated=self.time_updated and self.time_updated.isoformat(), ) - def flag_data_modified(self): # seql alchemy isnt reliable in recognising modifications to JSON, so we explicitely tell it orm.attributes.flag_modified(self, 'data') class BaseModelWithExternal(BaseModel): - __abstract__ = True ext_id: str = db.Column(db.String(64), nullable=True) ext_src: str = db.Column(db.String(32), nullable=True) - #Example, for MTurk, ext_src=mturk; + # Example, for MTurk, ext_src=mturk; # ext_id= workerId for user, # HITID for ChatTopic, # AssignmentID for ChatThread def as_dict(self) -> Dict[str, Any]: - return super().as_dict() | dict(ext_id = self.ext_id, ext_src = self.ext_src) - + return super().as_dict() | dict(ext_id=self.ext_id, ext_src=self.ext_src) class User(BaseModelWithExternal): - __tablename__ = 'user' ANONYMOUS = 'Anonymous' @@ -100,7 +106,6 @@ class User(BaseModelWithExternal): # eg: bot, human, admin role: str = db.Column(db.String(30), nullable=True) - @property def is_active(self): return self.active @@ -108,7 +113,7 @@ def is_active(self): @property def is_authenticated(self): return self.is_active - + @property def is_admin(self): return self.role == self.ROLE_ADMIN @@ -135,7 +140,7 @@ def _hash(cls, secret): return hashlib.sha3_256(secret.encode()).hexdigest() def verify_secret(self, secret): - if not self.secret: # empty secret => login disabled + if not self.secret: # empty secret => login disabled return False return self.secret == self._hash(secret) @@ -163,19 +168,18 @@ def create_new(cls, id: str, secret: str, name: str = None, def as_dict(self): return super().as_dict() | dict( id=self.id, - name= self.name, + name=self.name, role=self.role, ) class ChatMessage(BaseModelWithExternal): - __tablename__ = 'message' id: int = db.Column(db.Integer, primary_key=True) text: str = db.Column(db.String(2048), nullable=False) - is_seed: bool = db.Column(db.Boolean) - + is_seed: bool = db.Column(db.Boolean) + user_id: str = db.Column( db.String(31), db.ForeignKey('user.id'), nullable=False) thread_id: int = db.Column( @@ -186,24 +190,27 @@ def time(self): return self.time_created def as_dict(self): - return super().as_dict() | dict( + return super().as_dict() | dict( text=self.text, - is_seed = self.is_seed, + is_seed=self.is_seed, user_id=self.user_id, thread_id=self.thread_id, ) + def save_message_to_db(self): + save_to_db(self) + + UserThread = db.Table( 'user_thread', db.Column('user_id', db.String(31), db.ForeignKey('user.id'), primary_key=True), db.Column('thread_id', db.Integer, db.ForeignKey('thread.id'), primary_key=True) - ) +) class ChatThread(BaseModelWithExternal): - __tablename__ = 'thread' id: int = db.Column(db.Integer, primary_key=True) @@ -235,9 +242,20 @@ class ChatThread(BaseModelWithExternal): need_moderator_bot = db.Column(db.Boolean, server_default=sql.expression.true(), nullable=False) + speak_order = db.Column(db.JSON(), nullable=False, server_default='[]') + + current_speaker = db.Column(db.String(64), nullable=True) + + current_speaker_idx = db.Column(db.Integer, nullable=False, server_default='0') + + current_turns = db.Column(db.Integer, nullable=False, server_default='0') + + remaining_turns = db.Column(db.Integer, nullable=True) + # We include the following rows because the topic may be deleted. # But we still need to see the content of one thread even if the corresponding # topic is gone. + bot_name: str = db.Column(db.String(64), nullable=True) engine: str = db.Column(db.String(64), nullable=True) persona_id: str = db.Column(db.String(64), nullable=True) max_threads_per_topic: int = db.Column(db.Integer, nullable=True) @@ -245,6 +263,167 @@ class ChatThread(BaseModelWithExternal): max_human_users_per_thread: int = db.Column(db.Integer, nullable=True) human_moderator: str = db.Column(db.String(32), nullable=True) reward: str = db.Column(db.String(32), nullable=True) + parameters: str = db.Column(db.JSON(), nullable=False, server_default='{}') + + __avoid_double_create = set() + + @classmethod + def get_thread_by_id(cls, thread_id: str) -> Optional['ChatThread']: + """ + Get a thread instance by thread id. + :param thread_id: the id of the thread + :return: the chat thread instance + """ + thread = ChatThread.query.get(thread_id) + return thread + + @classmethod + def get_next_vacancy_thread_for_topic(cls, topic: 'ChatTopic') -> Optional['ChatThread']: + """ + Get a thread instance that has vacancy. + :return: the chat thread instance + """ + threads = ChatThread.query.filter( + ChatThread.topic_id == topic.id + ).all() + for thread in threads: + if len([u for u in thread.users + if u.role in [User.ROLE_HUMAN, User.ROLE_HUMAN_MODERATOR]]) < thread.max_human_users_per_thread: + return thread + + @classmethod + def get_thread_by_topic_and_user(cls, topic: 'ChatTopic', user: User) -> Optional['ChatThread']: + """ + Query a thread instance of a topic that the user can be in (or already in). + :param topic: the topic of the chat thread + :param user: the user that the chat thread involves + :return: a chat thread instance + """ + threads = ChatThread.query.filter( + ChatThread.topic_id == topic.id + ).all() + for thread in threads: + if user in thread.users: + return thread + + @classmethod + def create_thread_of_topic(cls, topic: 'ChatTopic', bot_name: str = 'gpt') -> Optional['ChatThread']: + """ + Create a new chat thread. + :param topic: the topic of the chat thread + :param bot_name: the bot to user + :return: a chat thread instance if success, or None + """ + from .utils import get_speak_order + + if topic.id in cls.__avoid_double_create: + return None + + cls.__avoid_double_create.add(topic.id) + + speak_order = get_speak_order(topic) + + thread = ChatThread( + topic_id=topic.id, + data=topic.data, + bot_name=bot_name, + engine=topic.endpoint, + persona_id=topic.persona_id, + max_turns_per_thread=topic.max_turns_per_thread, + human_moderator=topic.human_moderator, + reward=topic.reward, + max_threads_per_topic=topic.max_threads_per_topic, + max_human_users_per_thread=topic.max_human_users_per_thread, + parameters=topic.parameters, + need_moderator_bot=bool(topic.human_moderator != 'yes'), + speak_order=speak_order, + current_speaker_idx=0, + current_speaker=speak_order[0], + remaining_turns=topic.max_turns_per_thread, + ) + + thread.thread_state = 1 + + db.session.add(thread) + db.session.flush() + + for cv in topic.data['conversation']: + text = cv['text'] + data = dict( + text_orig=cv.get('text_orig'), + speaker_id=cv.get('speaker_id'), + fake_start=True + ) + msg = ChatMessage(thread_id=thread.id, user_id='context', is_seed=True, text=text, data=data) + save_to_db(msg) + thread.messages.append(msg) + + thread.thread_state = 2 + + db.session.commit() + + cls.__avoid_double_create.remove(topic.id) + return thread + + def set_external_info_for_user(self, user: User, ext_id=None, ext_src=None, data=None): + if self.ext_id is None: + self.ext_id = ext_id + + if self.ext_src is None: + self.ext_src = ext_src + + if self.assignment_id_dict is None: + self.assignment_id_dict = {} + + self.assignment_id_dict[user.id] = ext_id + + if self.submit_url_dict is None: + self.submit_url_dict = {} + + if data and data.get(ext_src) is not None: + self.submit_url_dict[user.id] = data.get(ext_src).get('submit_url') + + if user not in self.users: + self.users.append(user) + + self.flag_speakers_modified() + self.flag_assignment_id_dict_modified() + self.flag_submit_url_dict_modified() + + def add_user(self, user: User, role: str = None): + if user not in self.users: + self.users.append(user) + if role is not None: + speaker_dict = dict(self.speakers) + speaker_dict[user.id] = role + self.speakers = speaker_dict + user.role = User.ROLE_HUMAN_MODERATOR if role == 'Moderator' else User.ROLE_HUMAN + + def append_message(self, message: ChatMessage) -> Tuple[bool, str]: + """ + Append a message to the thread. Maintain the thread's current speaker. + :param message: the message to append + :return: a tuple of (success, info) + """ + if message.user_id not in [u.id for u in self.users]: + return False, f'User {message.user_id} not part of thread {self.id}' + if message.data['speaker_id'] != self.current_speaker: + return False, f'User {message.user_id} is not current speaker ({self.current_speaker})' + + # Update thread state + self.current_speaker_idx += 1 + if self.current_speaker_idx >= len(self.speak_order): + self.current_speaker_idx = 0 + self.current_turns += 1 + self.remaining_turns -= 1 + if self.current_turns >= self.max_turns_per_thread: + self.episode_done = True + + self.current_speaker = self.speak_order[self.current_speaker_idx] + + self.messages.append(message) + + return True, 'Success' def count_turns(self, user: User): return sum(msg.user_id == user.id for msg in self.messages) @@ -266,13 +445,25 @@ def as_dict(self): episode_done=self.episode_done, users=[u.as_dict() for u in self.users], messages=[m.as_dict() for m in self.messages], - speakers=self.speakers + speakers=self.speakers, + current_speaker=self.current_speaker, + current_turns=self.current_turns, + speak_order=self.speak_order, + current_speaker_idx=self.current_speaker_idx, + need_moderator_bot=self.need_moderator_bot, + max_turns_per_thread=self.max_turns_per_thread, ) - + @property def socket_name(self): return f'sock4thread_{self.id}' + def update_thread_in_db(self): + update_in_db(self) + + def save_thread_to_db(self): + save_to_db(self) + class SuperTopic(BaseModelWithExternal): """ @@ -308,32 +499,45 @@ class ChatTopic(BaseModelWithExternal): max_human_users_per_thread: int = db.Column(db.Integer, nullable=False) human_moderator: str = db.Column(db.String(32), nullable=True) reward: str = db.Column(db.String(32), nullable=False) + parameters: str = db.Column(db.JSON(), nullable=False, server_default='{}') def as_dict(self): return super().as_dict() | dict(name=self.name) @classmethod def create_new(cls, super_topic: SuperTopic, endpoint, persona_id, max_threads_per_topic, - max_turns_per_thread, max_human_users_per_thread, human_moderator, reward): + max_turns_per_thread, max_human_users_per_thread, human_moderator, reward, parameters): cur_task_id = super_topic.next_task_id cur_id = f'{super_topic.id}_{cur_task_id:03d}' cur_name = f'{super_topic.name}_{cur_task_id:03d}' - + # update target user ids: - - + topic = ChatTopic(id=cur_id, name=cur_name, data=super_topic.data, super_topic_id=super_topic.id, ext_id=super_topic.ext_id, ext_src=super_topic.ext_src, endpoint=endpoint, persona_id=persona_id, max_threads_per_topic=max_threads_per_topic, max_turns_per_thread=max_turns_per_thread, max_human_users_per_thread=max_human_users_per_thread, human_moderator=human_moderator, - reward=reward) + reward=reward, + parameters=parameters) # log.info(f'Creating New Task {topic.id}') super_topic.next_task_id += 1 db.session.add(topic) db.session.commit() return cls.query.get(topic.id) - - + @classmethod + def get_topics_user_not_done(cls, user) -> List['ChatTopic']: + """ + Get all topics that user has not done + :param user: + :return: + """ + topics = ChatTopic.query.all() + res = [] + for topic in topics: + thread = ChatThread.get_thread_by_topic_and_user(topic, user) + if thread is None: + res.append(topic) + return res diff --git a/boteval/service.py b/boteval/service.py index 2515a3e..375465d 100644 --- a/boteval/service.py +++ b/boteval/service.py @@ -1,4 +1,6 @@ import os +import random +import threading from pathlib import Path import json from typing import List, Mapping, Optional, Tuple, Union @@ -18,7 +20,7 @@ from .bots import BotAgent, load_bot_agent from .transforms import load_transforms, Transforms from .mturk import MTurkService - +from .utils import get_next_human_role class ChatManager: @@ -70,7 +72,7 @@ def init_chat_context(self, thread: ChatThread): log.info(f'{thread.id} has no messages, so nothing to init') return log.info(f'Init Thread ID {thread.id}\'s context with {len(thread.messages)} msgs') - topic_appeared = False + topic_appeared = False messages = [msg.as_dict() for msg in thread.messages] self.bot_agent.init_chat_context(messages) @@ -201,7 +203,7 @@ def __init__(self, config: TaskConfig, task_dir:Path, self.bot_transforms = load_transforms(transforms_conf['bot']) self.exporter = FileExportService(self.resolve_path(config.get('chat_dir'), 'data')) - bot_name = config['chatbot']['bot_name'] + self.bot_name = config['chatbot']['bot_name'] # bot_args are no longer used as we always load all possible bots and chose the one we need at launching. bot_args = config['chatbot'].get('bot_args') or {} @@ -217,18 +219,19 @@ def __init__(self, config: TaskConfig, task_dir:Path, self.persona_id_list = [x['id'] for x in persona_jsons] # Initialize all possible bots - self.bot_agent_dict = {} - for cur_endpoint_name in self.endpoints: - for cur_persona_id in self.persona_id_list: - tmp_dict = { - # 'engine': cur_engine_name, - 'default_endpoint': cur_endpoint_name, - 'persona_id': cur_persona_id - } - self.bot_agent_dict[(cur_endpoint_name, cur_persona_id)] = load_bot_agent(bot_name, tmp_dict) + # self.bot_agent_dict = {} + # for cur_endpoint_name in self.endpoints: + # for cur_persona_id in self.persona_id_list: + # tmp_dict = { + # # 'engine': cur_engine_name, + # 'default_endpoint': cur_endpoint_name, + # 'persona_id': cur_persona_id + # } + # self.bot_agent_dict[(cur_endpoint_name, cur_persona_id)] = load_bot_agent(bot_name, tmp_dict) # self.persona_id = bot_args.get('persona_id') # self.bot_agent = load_bot_agent(bot_name, bot_args) + self.cur_bot_agent = None self.limits = config.get('limits') or {} self.ratings = config['ratings'] @@ -249,7 +252,7 @@ def check_ext_url(self, ping_url, wait_time=C.PING_WAIT_TIME): # this will be called by app hook before_first_request if not self.config['flask_config'].get('SERVER_NAME'): log.warning('flask_config.SERVER_NAME is not set. crowd launching feature is disabled') - self._external_url_ok = None + self._external_url_ok = True return log.info(f"Pinging URL {ping_url} in {wait_time} secs") @@ -371,7 +374,7 @@ def init_db(self, init_topics=True): # db.session.commit() def create_topic_from_super_topic(self, super_topic_id, endpoint, persona_id, max_threads_per_topic, - max_turns_per_thread, max_human_users_per_thread, human_moderator, reward): + max_turns_per_thread, max_human_users_per_thread, human_moderator, reward, parameters): """ Create a topic from a super topic The terminology is confusing. A super topic is a topic in the old version. @@ -382,7 +385,7 @@ def create_topic_from_super_topic(self, super_topic_id, endpoint, persona_id, ma max_threads_per_topic=max_threads_per_topic, max_turns_per_thread=max_turns_per_thread, max_human_users_per_thread=max_human_users_per_thread, - human_moderator=human_moderator, reward=reward) + human_moderator=human_moderator, reward=reward, parameters=parameters) db.session.add(new_topic) db.session.commit() @@ -448,169 +451,35 @@ def get_thread_for_topic(self, user, topic: ChatTopic, create_if_missing=True, create a new thread if create_if_missing is True. """ - # log.info('data is: ', data) - log.info(f'topic.human_moderator is: {topic.human_moderator}') - - if topic.human_moderator == 'yes' and data is not None and data.get(ext_src) is not None: - cur_user_is_qualified = self.crowd_service.is_worker_qualified(user_worker_id=user.id, - qual_name='human_moderator_qualification') - - if cur_user_is_qualified: - log.info(f"Assign human moderator role to worker_id: {user.id}") - user.role = User.ROLE_HUMAN_MODERATOR - else: - log.info(f"Not Assign human moderator role to worker_id: {user.id}") - - topic_threads = ChatThread.query.filter_by(topic_id=topic.id).all() - # TODO: appply this second filter directly into sqlalchemy - thread = None - for tt in topic_threads: - if any(user.id == tu.id for tu in tt.users): - log.info('Topic thread already exists; reusing it') - thread = tt - break - - # if tt.human_user_2 is None or tt.human_user_2 == '': - human_moderators = [user for user in tt.users if user.role == User.ROLE_HUMAN_MODERATOR] - humans = [user for user in tt.users if user.role == User.ROLE_HUMAN] - - # if tt.need_moderator_bot and user.role == User.ROLE_HUMAN_MODERATOR: - # print("Current chat thread does not need a human moderator, topic id: ", topic.id, ' user.role:', user.role) - # continue - - if topic.human_moderator == 'yes' and len(human_moderators) > 0 and user.role == User.ROLE_HUMAN_MODERATOR: - log.info("More than one human moderator not allowed, topic id: ", topic.id) - continue - - if len(humans) + len(human_moderators) < topic.max_human_users_per_thread: - # Mark the thread as "is being created". - # This is to prevent other users from joining the thread at the same time. - if tt.thread_state == 1: - log.error("thread is being created") - return None - - log.info('human_user_2 join thread!') - - # store speakers id - chat_topic = ChatTopic.query.get(tt.topic_id) - loaded_users = [speaker_id for speaker_id in chat_topic.data['conversation']] - speakers = [cur_user.get('speaker_id') for cur_user in loaded_users] - - if topic.human_moderator == 'yes' and user.role == User.ROLE_HUMAN_MODERATOR: - # tt.need_moderator_bot = False - tt.speakers[user.id] = 'Moderator' - elif topic.human_moderator == 'yes' and len(human_moderators) == 1: - tt.speakers[user.id] = speakers[-1] - else: - i = -2 - while len(speakers) + i >= 0: - if speakers[i] != speakers[-1]: - # user.name = speakers[i] - tt.speakers[user.id] = speakers[i] - # tt.user_2nd = user.id - # tt.speaker_2nd = speakers[i] - break - else: - i -= 1 - - tt.assignment_id_dict[user.id] = ext_id - if tt.data.get(ext_src) is not None: - tt.submit_url_dict[user.id] = tt.data.get(ext_src).get('submit_url') - - # user.name = speakers[-2] - log.info(f'2nd user is: {user.id}, 2nd speaker is: {tt.speakers[user.id]}') - - tt.users.append(user) - # tt.users.append(self.bot_user) - # tt.users.append(self.context_user) - # tt.human_user_2 = user.id - - tt.flag_speakers_modified() - tt.flag_assignment_id_dict_modified() - tt.flag_submit_url_dict_modified() - db.session.merge(tt) - db.session.flush() - db.session.commit() + thread = (ChatThread.get_thread_by_topic_and_user(topic, user) or + ChatThread.get_next_vacancy_thread_for_topic(topic)) + if thread is not None: + if user not in thread.users: + role = get_next_human_role(user, thread, topic) + thread.add_user(user, role) + thread.update_thread_in_db() + if ext_id or ext_src or data: + thread.set_external_info_for_user(user, ext_id , ext_src, data) + thread.update_thread_in_db() + return thread + elif create_if_missing: + log.info(f'creating a thread: user: {user.id} topic: {topic.id}') - thread = tt - break + thread = ChatThread.create_thread_of_topic(topic, self.bot_name) + if not thread: + return None - if not thread and create_if_missing: - log.info(f'creating a thread: user: {user.id} topic: {topic.id}') - data = data or {} - # If there is no data from input, we directly use the data from the topic - # If there is data, we shouldn't update it with the topic data, otherwise the 'ext_src' might be overridden - if not data: - data.update(topic.data) - thread = ChatThread(topic_id=topic.id, ext_id=ext_id, ext_src=ext_src, data=data, engine=topic.endpoint, - persona_id=topic.persona_id, max_threads_per_topic=topic.max_threads_per_topic, - max_turns_per_thread=topic.max_turns_per_thread, human_moderator=topic.human_moderator, - reward=topic.reward, max_human_users_per_thread=topic.max_human_users_per_thread) - - chat_topic = ChatTopic.query.get(thread.topic_id) - loaded_users = [speaker_id for speaker_id in chat_topic.data['conversation']] - speakers = [cur_user.get('speaker_id') for cur_user in loaded_users] - - if thread.human_moderator == 'yes': - thread.need_moderator_bot = False - log.info(topic.id, 'does not need a moderator bot') - else: - thread.need_moderator_bot = True - log.info(topic.id, 'needs a moderator bot') + role = get_next_human_role(user, thread, topic) - # user.name = speakers[-1] - if thread.speakers is None: - thread.speakers = {} + thread.add_user(user, role) + thread.add_user(self.bot_user) + thread.add_user(self.context_user) - if thread.assignment_id_dict is None: - thread.assignment_id_dict = {} + thread.set_external_info_for_user(user, ext_id, ext_src, data) - if thread.submit_url_dict is None: - thread.submit_url_dict = {} + thread.update_thread_in_db() - if topic.human_moderator == 'yes' and user.role == User.ROLE_HUMAN_MODERATOR: - # thread.need_moderator_bot = False - thread.speakers[user.id] = 'Moderator' - else: - thread.speakers[user.id] = speakers[-1] - # thread.user_1st = user.id - # thread.speaker_1st = speakers[-1] - - thread.assignment_id_dict[user.id] = ext_id - if data.get(ext_src): - thread.submit_url_dict[user.id] = data.get(ext_src).get('submit_url') - - thread.thread_state = 1 - log.info("Set thread state to 1") - log.info(f'1st user is: {user.id}, 1st speaker is: {thread.speakers[user.id]}') - - thread.users.append(user) - thread.users.append(self.bot_user) - thread.users.append(self.context_user) - - thread.human_user_1 = user.id - - db.session.add(thread) - db.session.flush() # flush it to get thread_id - for m in topic.data['conversation']: - # assumption: messages are pre-transformed to reduce wait times - text = m['text'] - data = dict(text_orig=m.get('text_orig'), - speaker_id=m.get('speaker_id'), - fake_start=True) - msg = ChatMessage(text=text, user_id=self.context_user.id, thread_id=thread.id, is_seed=True, data=data) - db.session.add(msg) - thread.messages.append(msg) - thread.thread_state = 2 - - # if moderator: - if topic.human_moderator == 'yes': - log.info("Set thread state to 2") - - db.session.merge(thread) - db.session.flush() - db.session.commit() - return thread + return thread def get_thread(self, thread_id) -> Optional[ChatThread]: result = ChatThread.query.get(thread_id) @@ -673,9 +542,19 @@ def update_thread_ratings(self, thread: ChatThread, ratings: dict, user_id: str) # @functools.lru_cache(maxsize=256) def get_dialog_man(self, thread: ChatThread) -> DialogBotChatManager: - cur_bot_agent = self.bot_agent_dict[(thread.engine, thread.persona_id)] + # cur_bot_agent = self.bot_agent_dict[(thread.engine, thread.persona_id)] + log.info(f'create bot agent with: {thread.parameters}') + parameters_dict = { + 'parameters': thread.parameters + } + + # bot is created by darma, paramter: dictionary + log.info(f'bot name is: {thread.bot_name}') + if self.cur_bot_agent is None: + self.cur_bot_agent = load_bot_agent(thread.bot_name, parameters_dict) + return DialogBotChatManager(thread=thread, - bot_agent=cur_bot_agent, + bot_agent=self.cur_bot_agent, max_turns=thread.max_turns_per_thread, bot_transforms=self.bot_transforms, human_transforms=self.human_transforms) @@ -697,11 +576,11 @@ def launch_topic_on_crowd(self, topic: ChatTopic): if not self.crowd_service: log.warning('Crowd service not configured') return None - if not self.is_external_url_ok: - msg = 'External URL is not configured correctly. Skipping.' - log.warning(msg) - flask.flash(msg) - return None + # if not self.is_external_url_ok: + # msg = 'External URL is not configured correctly. Skipping.' + # log.warning(msg) + # flask.flash(msg) + # return None if self.crowd_name in (C.MTURK, C.MTURK_SANDBOX): landing_url = flask.url_for('app.mturk_landing', topic_id=topic.id, _external=True, _scheme='https') diff --git a/boteval/static/css/chatui.css b/boteval/static/css/chatui.css new file mode 100644 index 0000000..4669147 --- /dev/null +++ b/boteval/static/css/chatui.css @@ -0,0 +1,32 @@ +.dot { + opacity: 0; + animation: showHideDot 2s ease-in-out infinite; + font-size: 1rem; +} + +.dot.one { + animation-delay: 0.2s; +} + +.dot.two { + animation-delay: 0.4s; +} + +.dot.three { + animation-delay: 0.6s; +} + +@keyframes showHideDot { + 0% { + opacity: 0; + } + 50% { + opacity: 1; + } + 60% { + opacity: 1; + } + 100% { + opacity: 0; + } +} \ No newline at end of file diff --git a/boteval/static/js/chatui.js b/boteval/static/js/chatui.js new file mode 100644 index 0000000..f73617a --- /dev/null +++ b/boteval/static/js/chatui.js @@ -0,0 +1,282 @@ +/** + * Add a message to the chat window + * @param msgID {string} the id of the message + * @param content {string} the content of the message, can be html + * @param userName {string} the name of the user who sent the message + * @param timeCreated {string} the time the message was created + * @param color {string} the color of the message, in ["secondary", "light", "dark", "info", "primary"] + * @param side {string} the side of the message, in ["left", "right"] + */ +function addDialogBox(msgID, content, userName, timeCreated, color, side = 'left') { + const chat_html = ` +
  • +
    +
    +

    ${userName}

    +

    ${timeCreated}

    +
    +
    + ${content} +
    +
    +
    • ` + $('#chat-thread').append($.parseHTML(chat_html)) +} + +/** + * Add a message to the chat window, automatically assign color and side based on the user + * @param msg message object, {data, user_id, time_created, text, ...} + * @param userIDs {string[]} the IDs of all users in the chat + * @param userRoles {string[]} the roles of all users in the chat + * @param curUserId {string} the ID of the current user + * @param curUserRole {string} the role of the current user, e.g. 'a', 'b', 'Moderator'... + */ +function addChatMessage(msg, userIDs, userRoles, curUserId, curUserRole) { + const colors = ["secondary", "light", "dark", "info", "primary"] + const userId = msg.user_id + const idx = userIDs.indexOf(userId) + const userRole = msg.data?.speaker_id + const msgUserDisplayText = userId === curUserId ? `Your reply as ${userRole}` : userRole + const extra = msg.data?.text_orig + const content = msg.text + const text_display = extra ? ` +
    + ${content} + ${extra} +
    + ` : content + if (userRole === curUserRole) { + // This message is from the role played by the current user + addDialogBox(msg.id, text_display, msgUserDisplayText, msg.time_created, 'danger', 'right') + } else { + // The message is from the other user + const color = (userRole === 'Moderator') ? 'success' : colors[idx % colors.length] + addDialogBox(msg.id, text_display, msgUserDisplayText, msg.time_created, color, 'left') + } +} + +/** + * Scroll the chat window to the bottom + */ +function scrollToBottom() { + const chat_thread = document.getElementById('chat-thread') + chat_thread.scrollTop = chat_thread.scrollHeight +} + +/** + * Play a sound to indicate a new message + */ +function playNewMessageSound() { + new Audio('/static/img/new_message_beep.mp3').play(); +} + +/** + * Show/Hide page elements based on the current state + * @param state {string} in ["wait", "chat", "end"] + */ +function doSetPageState(state) { + if (state === 'wait') { + $('#end_info').hide() + $('#next_msg_form').hide() // dont let human reply + $('#waiting_info').show() // waiting animation show + } else if (state === 'chat') { + $('#end_info').hide() + $('#next_msg_form').show() // let human reply + $('#waiting_info').hide() // waiting animation hide + } else if (state === 'end') { + $('#end_info').show() + $('#next_msg_form').hide() + $('#waiting_info').hide() + //if (curUser.role !== 'Moderator') { + $('#ratings-view').show() + //} + } +} + +/** + * Refresh and put all the message on the screen + * @param thread the chat thread json object + */ +function loadChatThreadOnScreen(thread) { + $("#chat-thread").empty(); + const userIds = [] + const userRoles = [] + Object.keys(thread.speakers).forEach(key => { + userIds.push(key) + userRoles.push(thread.speakers[key]) + }) + for (const msg of thread.messages) { + addChatMessage(msg, userIds, userRoles, curUser.id, curUser.role) + } + // set remaining turns + $('#remaining-turns-count').text(thread.max_turns_per_thread - thread.current_turns) +} + +const threadId = document.getElementById('thread-id').textContent; + +let curUser = { + id: document.getElementById('user-id').textContent, + role: document.getElementById('user-role').textContent, +} + +const threadFetchUrl = document.getElementById('thread-object-url').href; +const submitUrl = document.getElementById('post-message-url').href; +const botReplyUrl = document.getElementById('bot-reply-url').href; + +function initChatThread() { + fetch(threadFetchUrl) + .then(response => response.json()) + .then(thread => { + loadChatThreadOnScreen(thread) + lastMessages = thread.messages.length + scrollToBottom() + setPageStateAccordingToThread(thread) + }) +} + +let avoidDuplicateReply = false; + +function tryGetBotReply(thread) { + if (avoidDuplicateReply) { + return + } + avoidDuplicateReply = true + $.post(botReplyUrl, { + thread_id: thread.id, + user_id: curUser.id, + turns: thread.current_turns, + speaker_idx: thread.current_speaker_idx, + }).done(reply => { + avoidDuplicateReply = false + }).fail(() => { + avoidDuplicateReply = false + }) +} + +/** + * Set the page state based on the current thread state. + * Try to get bot reply only if it's my turn next. + * @param thread + */ +function setPageStateAccordingToThread(thread) { + if (thread.episode_done) { + doSetPageState('end') + stopWaitingForNextTurn() + } else if (thread.current_speaker === curUser.role) { + doSetPageState('chat') + stopWaitingForNextTurn() + } else { + // wait + if (thread.need_moderator_bot && thread.current_speaker === 'Moderator') { + const next_speaker_idx = (thread.current_speaker_idx + 1) % thread.speak_order.length + if (thread.speak_order[next_speaker_idx] === curUser.role) { + // it's my turn next + tryGetBotReply(thread) + } + } + doSetPageState('wait') + startWaitingForNextTurn() + } +} + +let lastMessages = 0; + +/** + * Check if there are new messages in the thread. + * @param thread + */ +function checkNewMessages(thread) { + if (thread.messages.length > lastMessages) { + lastMessages = thread.messages.length; + playNewMessageSound() + scrollToBottom() + } +} + +let checkThreadId = -1; + +/** + * This function is called periodically to check if current chat thread is updated or not. + * Fetch the thread object from the server. + * 1. It updates new messages on the screen. + * 2. It sets the page state to 'chat', 'wait' or 'end' depending on the current turn. + * + */ +function threadCheckHandler() { + fetch(threadFetchUrl) + .then(response => response.json()) + .then(thread => { + loadChatThreadOnScreen(thread) + checkNewMessages(thread) + setPageStateAccordingToThread(thread) + }) +} + +/** + * Start a periodical timer to check if the thread is updated. + * @param delay {number} the delay in seconds + */ +function startWaitingForNextTurn(delay = 5) { + if (checkThreadId !== -1) { + return + } + checkThreadId = window.setInterval(threadCheckHandler, delay * 1000) +} + +/** + * Stop the periodical. + */ +function stopWaitingForNextTurn() { + if (checkThreadId === -1) { + return + } + window.clearInterval(checkThreadId) + checkThreadId = -1 +} + +function onButtonClicked(event) { + event.preventDefault(); + const content = $("#next_msg_text").val().trim() + if (!content) { + return + } + // submit new message + const newMessage = { + thread_id: threadId, + text: content, + speaker_id: curUser.role, + user_id: curUser.id, + } + $.post(submitUrl, newMessage).done(reply => { + addDialogBox( + reply.message_id, content, curUser.role, reply.timestamp, 'danger', 'right' + ) + lastMessages += 1 + scrollToBottom() + $('#next_msg_text').val('') // empty the input box + doSetPageState('wait') + startWaitingForNextTurn() + }).fail(() => { + alert('Something went wrong. Could not send message.') + }) +} + + +function chatUIProcess() { + $.ajaxSetup({ + crossDomain: true, + xhrFields: { + withCredentials: true + }, + }); + initChatThread() + // submit on enter key + $("#next_msg_text").keyup(event => { + if (event.which === 13) { + $("#next_msg_form").submit(); + } + }); + $("#next_msg_form").submit(onButtonClicked); +} + +window.onload = chatUIProcess; \ No newline at end of file diff --git a/boteval/templates/about.html b/boteval/templates/about.html deleted file mode 100644 index dc44cf3..0000000 --- a/boteval/templates/about.html +++ /dev/null @@ -1,31 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} -
    -
    -

    {% block title %} About {% endblock %}

    -
    - Chat Evaluation UI -
      - {% set repo_url="https://github.com/thammegowda/boteval"%} -
    • {{repo_url}}
      • -
    -
    - - {% if sys_info %} -
    -

    System Info

    - - {% for key, value in sys_info.items() %} - - - - - {% endfor %} -
    {{ key }} {{ value }}
    - -
    - {%endif%} -
    - -{% endblock %} \ No newline at end of file diff --git a/boteval/templates/admin/config.html b/boteval/templates/admin/config.html deleted file mode 100644 index 98b4d0c..0000000 --- a/boteval/templates/admin/config.html +++ /dev/null @@ -1,11 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} - - - -
    {{config_yaml|safe}}
    - - -{%endblock%} diff --git a/boteval/templates/admin/index.html b/boteval/templates/admin/index.html deleted file mode 100644 index 2a4a17f..0000000 --- a/boteval/templates/admin/index.html +++ /dev/null @@ -1,45 +0,0 @@ -{% extends 'base.html' %} - - -{% block content %} -
    -

    {% block title %} Chat Admin {% endblock %}

    -
    -
    -
      -
    • - Topics - {{counts.get('topic')}} -
      • -
    • - Users - {{counts.get('user')}} -
      • -
    • - Threads - {{counts.get('thread')}} -
      • -
    • - Messages - {{counts.get('message')}} -
      • -
    -
    - -
    - {% if crowd_name %} -
    -
    -

    Endpoint: {{mturk_endpoint_url}}

    -
    -
    - - {%endif%} -
    -
    -
    -{% endblock %} \ No newline at end of file diff --git a/boteval/templates/admin/mturk/HIT.html b/boteval/templates/admin/mturk/HIT.html deleted file mode 100644 index 849bf72..0000000 --- a/boteval/templates/admin/mturk/HIT.html +++ /dev/null @@ -1,33 +0,0 @@ -{% extends 'base.html' %} -{% from 'admin/mturk/_macros.html' import render_assignment %} - -{% block content %} -{% set mturk_where = meta['mturk_where'] %} - -
    - -
    -
    -

    Assignments

    - -
      - {% set index = 0 %} - {% for asgn in data['Assignments'] %} -
    1. - {{ render_assignment(asgn, base_pay, pay_per_hour, bonus_pay[index], qtypes, crowd_name=meta['crowd_name'])}} - {% set index = index + 1 %} -
      2. - {% endfor %} -
    -
    -
    -
    -{% endblock %} \ No newline at end of file diff --git a/boteval/templates/admin/mturk/HITs.html b/boteval/templates/admin/mturk/HITs.html deleted file mode 100644 index 33c65db..0000000 --- a/boteval/templates/admin/mturk/HITs.html +++ /dev/null @@ -1,52 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} -{% set mturk_where = meta['mturk_where'] %} -
    - -
    -
    -

    HITs

    - -
      - {% for HIT in data['HITs'] %} - {% set hit_id = HIT['HITId'] %} -
    • - {{ loop.index }} ― - - - {{ hit_id }} | {{ HIT['Title'] }}
      -
      - Status:{{ HIT['HITStatus'] }} {{ HIT['HITReviewStatus'] }}⤵ - {% for key, value in HIT.items() %} -
        -
      • {{key}}: {{value}}
        • -
      - {% endfor %} -
      -
      - {% if HIT['HITStatus'] == 'Assignable' %} - - {%else%} - - {%endif%} - -
      - -
      • - {% endfor %} - -
    -
    -
    -
    -{% endblock %} \ No newline at end of file diff --git a/boteval/templates/admin/mturk/_macros.html b/boteval/templates/admin/mturk/_macros.html deleted file mode 100644 index 4869b32..0000000 --- a/boteval/templates/admin/mturk/_macros.html +++ /dev/null @@ -1,62 +0,0 @@ -{% macro render_assignment(asgn, base_pay, pay_per_hour, bonus_pay, qtypes=None, crowd_name='mturk') %} -{{asgn['AssignmentId']}} - [{{'sandbox' if 'sandbox' in crowd_name else 'live'}}] -
      -
    • Worker: {{asgn['WorkerId']}} | Status: {{asgn['AssignmentStatus']}}
      • -
    • HIT: {{asgn['HITId']}}
      • -
    • Submitted on {{asgn['SubmitTime']}}
      • - {% if 'ApprovalTime' in asgn %}
    • Approved on {{asgn['ApprovalTime']}}
      • {%endif%} - {% if 'AcceptTime' in asgn %}
    • Accepted on {{asgn['AcceptTime']}}
      • {%endif%} - {% if 'AcceptTime' in asgn %}
    • Total time spent on hit (h:mm:ss): {{asgn['SubmitTime'] - asgn['AcceptTime']}}
      • {%endif%} -
    -{% if asgn['AssignmentStatus'] == 'Submitted' %} -
    - - -
    -{% elif asgn['AssignmentStatus'] == 'Approved' %} -

    (Already approved)

    -{% else %} -

    (Approval is not applicable)

    -{% endif %} - - - -{% if asgn['AssignmentStatus'] == 'Approved' %} -
    - -

    Base pay given to worker for completing task: {{base_pay}}

    -

    Hourly rate of pay desired for worker: {{pay_per_hour}} dollars/hour

    -

    Bonus to be given: ${{bonus_pay}}

    -
    - - -
    -

    -
    -{% endif %} - -{% if qtypes %} -
    -
    - - -
    - -
    -{% else %} -

    (No qualifications are available)

    -{%endif%} - -{% endmacro %} diff --git a/boteval/templates/admin/mturk/home.html b/boteval/templates/admin/mturk/home.html deleted file mode 100644 index 18d57d2..0000000 --- a/boteval/templates/admin/mturk/home.html +++ /dev/null @@ -1,27 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} -{% set mturk_where = meta['mturk_where'] %} -
    - -
    -
    -
    -
    -

    Endpoint: {{meta['mturk_endpoint_url']}}

    -
    -
    - -
    -
    - -{% endblock %} \ No newline at end of file diff --git a/boteval/templates/admin/mturk/qualification.html b/boteval/templates/admin/mturk/qualification.html deleted file mode 100644 index 2d9bb84..0000000 --- a/boteval/templates/admin/mturk/qualification.html +++ /dev/null @@ -1,60 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} -{% set mturk_where = meta['mturk_where'] %} -
    - -
    -
    -

    HITs

    - -
      - {% for HIT in data['HITs'] %} - {% set hit_id = HIT['HITId'] %} -
    • - {{ loop.index }} ― - - - {{ hit_id }} | {{ HIT['Title'] }} -
      - ⤵ - {% for key, value in HIT.items() %} -
        -
      • {{key}}: {{value}}
        • -
      - {% endfor %} -
      - -
      • - {% endfor %} - -
    -
    -
    -

    Workers

    - -
      - {% for worker in data['workers'] %} - {% set worker_id = worker['WorkerId'] %} -
    1. - {{worker_id}} - Status: {{worker['Status']}} | {{worker['GrantTime']}} - -
      2. - {% endfor %} -
    -
    - -
    -
    -{% endblock %} \ No newline at end of file diff --git a/boteval/templates/admin/mturk/qualifications.html b/boteval/templates/admin/mturk/qualifications.html deleted file mode 100644 index 6255058..0000000 --- a/boteval/templates/admin/mturk/qualifications.html +++ /dev/null @@ -1,46 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} -{% set mturk_where = meta['mturk_where'] %} -
    - -
    -
    -
    -
    -

    Endpoint: {{meta['mturk_endpoint_url']}}

    - -
    -
    - -
      - {% for qtype in qtypes %} - {% set qual_id = qtype['QualificationTypeId'] %} -
    • - {{loop.index}} ― - {{ qual_id }} | {{ qtype['Name'] }} - - -
      - ⤵ -
        - {% for key, value in qtype.items() if key != 'QualificationTypeId' %} -
      • {{key}}: {{value}}
        • - {% endfor %} -
      -
      -
      • - {% endfor %} - -
    -
    -
    -
    - -{% endblock %} \ No newline at end of file diff --git a/boteval/templates/admin/threads.html b/boteval/templates/admin/threads.html deleted file mode 100644 index d37ebe7..0000000 --- a/boteval/templates/admin/threads.html +++ /dev/null @@ -1,54 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} -
    - - -

    {% block title %} Chat Admin {% endblock %}

    -
    -
    -
    -
    - Found {{threads | length }} threads -
    -
    - - - - - - - - - - - {% for thread in threads %} - - - - - - - - - - {% endfor %} -
    IDTopicUsersEpisode | RatingsTimeActionsExt
    {{thread.id}} {{thread.topic_id}} {% for user in thread.users %} {{user.id}}, {%endfor%} {{thread.episode_done}}, {{thread.data.get('rating_done', '')}} {{thread.time_created|ctime}} {{thread.time_modified|ctime}} - {% if thread.ext_id %} -
      -
    • {{thread.ext_src or ''}} ID: {{thread.ext_id or ''}}
      • -
    - {%else%} N/A {% endif%} -
    -
    -
    -
    -{% endblock %} \ No newline at end of file diff --git a/boteval/templates/admin/topics.html b/boteval/templates/admin/topics.html deleted file mode 100644 index 0b00919..0000000 --- a/boteval/templates/admin/topics.html +++ /dev/null @@ -1,248 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} -
    - - -

    {% block title %} Topics {% endblock %}

    -
    -
    - {%set ext_url= url_for('app.index', _external=True, _scheme='https')%} - External URL: {{ext_url}} Okay? {{external_url_ok and 'Yes' or 'No'}} -
    - If you are planning to crowdsource tasks on platforms like MTurk, make sure to have the above URL is accessible via the (public) Internet.
    -
    - More Info: -
      -
    1. set flask_config.SERVER_NAME= in config YML file to publicly visible server name/address.
      2. -
    2. Install and verify your SSL certificate are correct for HTTPS connection. Example: https://certbot.eff.org/ makes it easy. However, you need a domain name for your server.
      3. -
    3. Setup reverse proxy to serve this app. Example: see docs/nginx-conf.adoc file in git repository for an example Nginx config.
      4. -
    -
    -
    -
    -
    -
    -
    -
    - - - - - - -
    -
    -
    - -
    - -
    - -
    -
    -

    Set maximum thread per user:

    -
    -
    - - -
    -
    - -
    -
    -
    -{# {% if crowd_name %}#} -
    -

    Multi-task Launch:

    -
    -
    - - -
    -
    - -
    -
    -
    -{# {% endif %}#} -     		-

    Multi-task Creation:

    -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - -
    -
    -
    -
    - Found {{ super_topics | length }} topics -
    -
    - - - - - - - - - {% for super_topic, num_threads in super_topics %} - - - - - - - - {% endfor %} -
    #TopicNum ThreadsTimeActions
    {{loop.index}} {{super_topic.id}} {{num_threads}} Created {{super_topic.time_created|ctime}}; {{super_topic.time_modified|ctime}} -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - - -
    -
    - -
    - -
    - {% for topic in super_topic.topics %} -
    Task ID: {{ topic.id }}
    -
      -
    • Num of Threads = {{ topic_thread_counts_dict.get(topic.id, 0) }}
      • -
    • Endpoint: {{ topic.endpoint }}
      • -
    • Persona_id: {{ topic.persona_id }}
      • -
    • Max threads per topic: {{ topic.max_threads_per_topic }}
      • -
    • Max turns per thread: {{ topic.max_turns_per_thread }}
      • -
    • Max human users per thread: {{ topic.max_human_users_per_thread }}
      • -
    • Human moderator: {{ topic.human_moderator }}
      • -
    • Reward: {{ topic.reward }}
      • - {% if topic.ext_id %} - {% set task_url = topic.data.get(topic.ext_src, {}).get('ext_url') %} -
    • {{topic.ext_src}}: {{topic.ext_id}}
      • -
    • - Task URL - -
      • - {% if topic.ext_src in [C.MTURK, C.MTURK_SANDBOX ] %} -
    • See MTurk Assignments
      • - {%endif%} - - {% elif crowd_name %} -
    • - Launch on {{crowd_name}} -
      • - {% else %} -
    • N/A
      • - {% endif %} -
    • - - Delete Task -
      • -
      -
    - {% endfor %} -
    -
    -
    -
    -{% endblock %} \ No newline at end of file diff --git a/boteval/templates/admin/users.html b/boteval/templates/admin/users.html deleted file mode 100644 index 1b9cee7..0000000 --- a/boteval/templates/admin/users.html +++ /dev/null @@ -1,43 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} -
    - - -

    {% block title %} Users {% endblock %}

    -
    -
    -
    -
    - Found {{users | length }} users -
    -
    - - - - - - - - - - {% for user in users %} - - - - - - - - - {% endfor %} -
    #IDNameJoinedLast ActiveExternal?
    {{loop.index}} {{user.id}} {{user.name}} {{user.time_created|ctime}} {{user.last_active|ctime}} {{user.ext_src or ''}} {{user.ext_id or '' }}
    -
    -
    -
    -{% endblock %} diff --git a/boteval/templates/base.html b/boteval/templates/base.html deleted file mode 100644 index f6f73b4..0000000 --- a/boteval/templates/base.html +++ /dev/null @@ -1,134 +0,0 @@ - - - - - - - - {% block title %} {% endblock %} - {% block head %} {% endblock %} - - - {% if environ['GA_GTAG'] is defined %} - - - {% endif %} - - - - - - - - - -{% set is_admin_user = cur_user and cur_user.is_authenticated and cur_user.role == 'admin' %} - - -
    - {%if not focus_mode %} - - {%endif%} -
    - {% with messages = get_flashed_messages() %} - {% if messages %} - - {% endif %} - {% endwith %} - {% block content %} {% endblock %} -
    - -
    -
    - - - {% if not focus_mode %} - - {% endif %} - - - - \ No newline at end of file diff --git a/boteval/templates/login.html b/boteval/templates/login.html deleted file mode 100644 index 5ed931e..0000000 --- a/boteval/templates/login.html +++ /dev/null @@ -1,105 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} -
    - -
    -
    -
    - -

    Please Sign In

    - Dont have an account? Click 'Sign Up' - - - {% if next %} - - {%endif%} - - -
    -
    -
    -
    - -

    Please Sign Up

    - - - {% if ext_id %} {%endif%} - {% if ext_src %} {%endif%} - - {% if onboarding %} - {%if onboarding.get('agreement_text') %} - - Please refresh if the agree and sign up button doesn't get you to the task. - {%endif%} - {%if onboarding.get('checkboxes') %} - {%for check_name, check_value in onboarding['checkboxes'].items() %} -
    - - -
    - {%endfor%} - {%endif%} - {%endif%} - - {% if next %} {%endif%} - - -
    -
    -
    -
    - -{% if onboarding and onboarding.get('agreement_text') %} - - -{%endif%} - - -{% endblock %} \ No newline at end of file diff --git a/boteval/templates/page.html b/boteval/templates/page.html deleted file mode 100644 index 8091118..0000000 --- a/boteval/templates/page.html +++ /dev/null @@ -1,5 +0,0 @@ -{% extends 'base.html' %} - -{% block content %} - {{content|safe}} -{%endblock%} diff --git a/boteval/templates/seamlesslogin.html b/boteval/templates/seamlesslogin.html deleted file mode 100644 index 2a7aa26..0000000 --- a/boteval/templates/seamlesslogin.html +++ /dev/null @@ -1,71 +0,0 @@ -{% set focus_mode = True %} -{% extends 'base.html' %} -{%block head%} - -{%endblock%} -{% block content %} -
    -
    - - {% if onboarding and onboarding.get('agreement_text') %} -
    - {{onboarding['agreement_text']|safe}} -
    - {%endif%} -
    -
    - - - - - - - {% if onboarding %} - {%if onboarding.get('checkboxes') %} - {%for check_name, check_value in onboarding['checkboxes'].items() %} -
    - - -
    - {%endfor%} - {%endif%} - {%endif%} - - {% if next %} {%endif%} - - - - - The server may take about 20 seconds to set up. Please wait patiently. -
    -
    -
    -
    - -{% endblock %} diff --git a/boteval/templates/user/_macros.html b/boteval/templates/user/_macros.html deleted file mode 100644 index e69de29..0000000 diff --git a/boteval/templates/user/chatui.html b/boteval/templates/user/chatui.html deleted file mode 100644 index 2858834..0000000 --- a/boteval/templates/user/chatui.html +++ /dev/null @@ -1,406 +0,0 @@ -{% extends "base.html" %} -{%block head%} - -{%endblock%} -{% block content %} -{% set reply_as_user = topic.data.get('target_user') %} -
    - {% if not focus_mode %} - - {%endif%} -
    -
    -
    - {% if reply_as_user %} -
    You are replying as Speaker - {{reply_as_user}}
    - {% endif %} - -
      -
      -
      - -
      - -
      -
      -
      - - -
      - Thanks for participating in the chat! Please submit your ratings to complete this task and receive compensation. -
      -
      - -
      - {{simple_instructions_html |safe}} -
      Your remaining turns: {{ remaining_turns }}
      - - {%if instructions_html %} - - {%endif%} - -
      0 %} style="display:none" {% endif %} - id="ratings-view"> - {% set is_disabled = 'disabled' if (thread.data or {}).get('rating_done') else '' %} -
      - - Thank you for your time. To complete this task, please answer the questions below: - {% set ratings_dict = (thread.data or {}).get('ratings') or {} %} - {% set prev_ratings = ratings_dict.get(cur_user.id) or {} %} -
        - {% for q in rating_questions %} -
      • - {{q['question']}} - {% if q['choices'] %} - - - - {% for ch in q['choices'] %} - - {%endfor%} - - {% endif %} - {% if q['range'] %} - - - - {% for i in range(q['range'][0], q['range'][1] + 1) %} - {% if q['range'][1] - q['range'][0] > 40 %} - {% if loop.index0 % 10 == 0 %} - - {% endif %} - {% else %} - - {% endif %} - {% endfor %} - - {% endif%} - {% if q['freetext'] %} - - {% endif %} -
        • - {% endfor %} -
      • - -
        • -
      - {% if focus_mode %} {%endif%} - -
      -
      -
      - -
      -
      -
      - - -
      - -{%if instructions_html %} - -{% endif %} -{% endblock%} \ No newline at end of file diff --git a/boteval/templates/user/chatui_two_users.html b/boteval/templates/user/chatui_two_users.html deleted file mode 100644 index e0fc1e3..0000000 --- a/boteval/templates/user/chatui_two_users.html +++ /dev/null @@ -1,662 +0,0 @@ -{% extends "base.html" %} -{%block head%} - -{%endblock%} -{% block content %} -{##} -{% set reply_as_user = thread.speakers[cur_user.id] %} - -
      - {% if not focus_mode %} - - {%endif%} -
      -
      -
      - {% if reply_as_user %} -
      You are replying as Speaker - {{reply_as_user}}
      - {% endif %} - -
        -
        -
        - -
        - -
        -
        -
        - - -
        - Thanks for participating in the chat! Please submit your ratings to complete this task and receive compensation. -
        -
        - -
        - {%if instructions_html %} - {{ simple_instructions_html | safe}} -
        Your remaining turns: {{ remaining_turns }}
        - - {%endif%} -
        0 %} style="display:none" {% endif %} - id="ratings-view"> - {% set is_disabled = 'disabled' if (thread.data or {}).get('rating_done') else '' %} -
        - - Thank you for your time. To complete this task, please answer the questions below: - {% set ratings_dict = (thread.data or {}).get('ratings') or {} %} - {% set prev_ratings = ratings_dict.get(cur_user.id) or {} %} -
          - {% for q in rating_questions %} -
        • - {{q['question']}} - {% if q['choices'] %} - - - - {% for ch in q['choices'] %} - - {%endfor%} - - {% endif %} - {% if q['range'] %} - - - - {% for i in range(q['range'][0], q['range'][1] + 1) %} - {% if q['range'][1] - q['range'][0] > 40 %} - {% if loop.index0 % 10 == 0 %} - - {% endif %} - {% else %} - - {% endif %} - {% endfor %} - - {% endif%} - {% if q['freetext'] %} - - {% endif %} -
          • - {% endfor %} -
        • - -
          • -
        - {% if focus_mode %} {%endif%} -
        - -
        -
        - -
        -
        -
        - - -
        - -{%if instructions_html %} - -{% endif %} -{% endblock%} \ No newline at end of file diff --git a/boteval/templates/user/index.html b/boteval/templates/user/index.html deleted file mode 100644 index 8de6ecb..0000000 --- a/boteval/templates/user/index.html +++ /dev/null @@ -1,41 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
        -
        - -

        Chat Topics

        - {% if limits %} - Summary -
          -
        • We have {{data | length}} chat topics in total.
          • -
        • You have started chatting on {{limits['threads_launched']}} topics. You may work on upto {{[data | length, limits['max_threads_per_user']]| min }} items below.
          • -
        • We call the chat thread is complete after certain turns (i.e., your replies). So far you have completed {{limits['threads_completed']}} of such.
          • -
        • We are trying to collect upto 10 (might be different for different tasks) threads per topic. You may not be able to start a chat if we have already received sufficient submissions.
          • -
        - {% endif %} -
          - - {% for topic, thread, n_threads in data %} -
        1. - {{topic['name']}} User Num: {{ topic.max_human_users_per_thread }} {{n_threads}}/{{ topic.max_threads_per_topic }}
          - {% if thread %} - {{ thread.time_created | ctime }} - {% if thread.episode_done %} - You have already completed this task. - No further action is necessary, though you may review your previous submission. - {% else %} - Continue You have not yet completed this task{%endif%} -
          - {% else %} - - Start chat -
          - {% endif %} -
          2. - {%endfor %} -
        -
        -
        - -{% endblock %} diff --git a/boteval/templates/user/mturk_submit.html b/boteval/templates/user/mturk_submit.html deleted file mode 100644 index 8db84cf..0000000 --- a/boteval/templates/user/mturk_submit.html +++ /dev/null @@ -1,26 +0,0 @@ -{% extends "base.html" %} - -{% block content %} -
        - {% set assignment_id = thread.assignment_id_dict[user.id] %} - {% set submit_url = thread.submit_url_dict[user.id] %} - -
        -

        Great job! This assignment is now complete.

        - One last thing..., please click "Inform .." button below to inform Mechanical Turk about the task completion. -

        -
        - - - - - -
        - -
        - Our team will review your work and issue the agreed upon reward. - For future communications with our team, please include the assignment ID {{assignment_id}}. -
        -
        - -{%endblock%} \ No newline at end of file diff --git a/boteval/utils.py b/boteval/utils.py index 4058986..e084af2 100644 --- a/boteval/utils.py +++ b/boteval/utils.py @@ -1,5 +1,6 @@ # import resource -from typing import Tuple +import json +from typing import Tuple, List import sys import time from datetime import datetime @@ -8,14 +9,14 @@ import flask_login as FL from . import log, C - +from .model import ChatThread, ChatTopic, User FLOAT_POINTS = 4 def render_template(*args, **kwargs): return flask.render_template(*args, environ=C.ENV, - cur_user=FL.current_user, C=C, **kwargs) + cur_user=FL.current_user, C=C, **kwargs) # def max_RSS(who=resource.RUSAGE_SELF) -> Tuple[int, str]: @@ -37,16 +38,15 @@ def render_template(*args, **kwargs): def format_bytes(bytes): - if bytes >= 10**6: - return f'{bytes/10**6:.2f} MB' - elif bytes >= 10**3: - return f'{bytes/10**3:.2f} KB' + if bytes >= 10 ** 6: + return f'{bytes / 10 ** 6:.2f} MB' + elif bytes >= 10 ** 3: + return f'{bytes / 10 ** 3:.2f} KB' else: return f'{bytes} B' def jsonify(obj): - if obj is None or isinstance(obj, (int, bool, str)): return obj elif isinstance(obj, float): @@ -57,7 +57,7 @@ def jsonify(obj): return [jsonify(it) for it in obj] elif hasattr(obj, 'as_dict'): return jsonify(obj.as_dict()) - #elif isinstance(ob, np.ndarray): + # elif isinstance(ob, np.ndarray): # return _jsonify(ob.tolist()) else: log.warning(f"Type {type(obj)} maybe not be json serializable") @@ -65,19 +65,17 @@ def jsonify(obj): def register_template_filters(app): - @app.template_filter('ctime') def timectime(s) -> str: if isinstance(s, datetime): return str(s) elif isinstance(s, int): - return time.ctime(s) # datetime.datetime.fromtimestamp(s) + return time.ctime(s) # datetime.datetime.fromtimestamp(s) elif s is None: return '' else: return str(s) - @app.template_filter('flat_single') def flatten_singleton(obj): res = obj @@ -88,4 +86,66 @@ def flatten_singleton(obj): res = obj[0] except: pass - return res \ No newline at end of file + return res + + +def get_speak_order(topic: ChatTopic) -> List[str]: + """ + Define the order of speakers in one turn for a topic. + This function is called when creating a new thread. + Customize this function to change the order of speakers. + The speaker order is the same as the order of the returned list. + + You can have a speaker speak multiple times in one turn by adding the speaker multiple times in the list. + For example, if you want to have User1 speak twice in one turn, you can return ['User1', 'User1', 'Moderator']. + + Chat room with existing conversation: the order is ['Moderator', 'b', 'Moderator', 'a', ...] + and User1 is the last speaker in the conversation, User2 is the second last speaker, etc. + :param human_users: number of human users in the chat room + :param topic: The topic of the thread + :return: A list of speaker ids + """ + if len(topic.data['conversation']) == 0: + # This is an empty chat room + return ['User1', 'Moderator'] + else: + # This is a chat room with existing conversation + all_roles = [m['speaker_id'] for m in reversed(topic.data['conversation']) if + m['speaker_id'] != 'Topic'] + speak_order = [] + human_moderators = int(topic.human_moderator == 'yes') + human_users = topic.max_human_users_per_thread - human_moderators + for role in all_roles[:human_users]: + if role not in speak_order: + speak_order.append('Moderator') + speak_order.append(role) + return speak_order + + +def get_next_human_role(user: User, thread: ChatThread, topic: ChatTopic) -> str: + """ + Assign a role to a user in a thread. + :param user: Currently unused + :param thread: The thread to assign role to + :param topic: The topic of the thread + :return: 'Moderator', 'User1', 'a', 'b', 'c', etc. + """ + has_moderator = thread.need_moderator_bot or any(thread.speakers[u] == 'Moderator' for u in thread.speakers.keys()) + + if topic.human_moderator == 'yes' and not has_moderator: + return 'Moderator' + + # Assign a speaker role + if len(topic.data['conversation']) == 0: + # This is an empty chat room + return 'User1' + else: + # This is a chat room with existing conversation + speaker_dict: dict = thread.speakers + existing_speakers = len([k for k, v in speaker_dict.items() if v != 'Moderator']) + all_roles = [m['speaker_id'] for m in reversed(topic.data['conversation']) if m['speaker_id'] != 'Topic'] + + assert existing_speakers < len(all_roles), \ + f'Error when assign speaker role. Existing speakers {existing_speakers} >= {len(all_roles)}' + + return all_roles[existing_speakers] diff --git a/docs/01-getting-started.adoc b/docs/01-getting-started.adoc index bb8f5c0..71016f7 100644 --- a/docs/01-getting-started.adoc +++ b/docs/01-getting-started.adoc @@ -106,6 +106,50 @@ example-chat-task/ <1> See <<#conf>> <2> See <<#add-bot>> +[#onboarding] += Onboarding +. Clone both `https://github.com/isi-nlp/isi_darma` and `https://github.com/isi-nlp/boteval` +. Create symlink from boteval/darma-task to isi_darma/boteval-darma-task +[source,shell] +---- +ln -s /isi_darma/boteval-darma-task darma-task +---- +[start=3] +. Create symlink from boteval/templates to isi_darma/configurable_files/templates +[source,shell] +---- +ln -s /isi_darma/configurable_files/templates boteval/templates +---- +[start=4] +. Run job locally +.. Create a python venv and install modules in requirements.txt +.. export `OPENAI_KEY=` +.. `python -m boteval darma-task -c darma-task/conf-local.yml -b /boteval -d` + +If you are using a windows machine: +Some of the setup steps are different: +[start=2] +. Create symlink from boteval/darma-task to isi_darma/boteval-darma-task + +[source,shell] +---- +mklink /J \boteval\darma-task \isi_darma\boteval-darma-task +---- +[start=3] +. Create symlink from boteval/boteval/templates to isi_darma/configurable_files/templates + +[source,shell] +---- + mklink /J \boteval\boteval\templates \isi_darma\configurable_files\templates +---- +[start=4] +. Run job locally +.. `.\venv\Scripts\activate` to activate your Python venv +.. Use the Command Prompt (cmd) instead of PowerShell: +`SET OPENAI_KEY=` (Use the keyword ‘SET’ instead of ‘export’ when setting up the OPENAI_KEY) + +All the other steps should be the same. + [#conf] = Config file @@ -118,8 +162,6 @@ chatbot: display_name: 'Moderator' topics_file: chat_topics.json bot_name: hf-transformers # <1> - bot_args: # <1> - model_name: facebook/blenderbot_small-90M onboarding: #<2> agreement_file: user-agreement.html @@ -146,6 +188,8 @@ limits: #<4> max_threads_per_topic: &max_assignments 3 max_turns_per_thread: 4 reward: &reward '0.01' # dollars + max_human_users_per_thread: 2 + human_moderator: 'yes' flask_config: #<5> # sqlalchemy settings https://flask-sqlalchemy.palletsprojects.com/en/2.x/config/ @@ -167,6 +211,22 @@ mturk: #<6> Title: 'Evaluate a chatbot' Keywords: 'chatbot,chat,research' Description: 'Evaluate a chat bot by talking to it for a while and receive a reward' + +customization: #<7> + argument1: + - option1 + - option2 + - option3 + argument2: 'default_value' + +limits_to_display: #<8> + - max_threads_per_user + - max_threads_per_topic + - multi_topics_creation + - multi_tasks_launch + - max_human_users_per_thread + - human_moderator + - max_turns_per_thread ---- :xrefstyle: full <1> See <<#conf-bot>> @@ -175,12 +235,13 @@ mturk: #<6> <4> See <<#conf-limits>> <5> See <<#conf-flask>> <6> See <<#conf-mturk>> +<7> See <<#conf-customization>> +<8> See <<#conf-limits_to_display>> [#conf-bot] == Bot Settings -* `bot_name` (str) and `bot_args` (dict) are required to enable chatbot backend. -* `bot_name` is a string where as `bot_args` is dictionary which is provided as key-word arguments. `bot_args` can be optional (i.e. missing) for bots that require no arguments. +* `bot_name` (str) is required to enable chatbot backend. .Example Bot [source,python,linenums] @@ -199,14 +260,12 @@ class TransformerBot(BotAgent): super().__init__(name=f'{self.NAME}:{model_name}') self.model_name = model_name ---- -Here, with `bot_name='hf-transformers'`, `bot_args` are optional as there are no arguments of `__init__` method that require value at runtime. However, if we want to change `model_name`, here is an example for how to provide it: +If we want to change `model_name`, here is an example for how to provide it: [source,yaml] ---- chatbot: #(other args) - bot_name: hf-transformers - bot_args: - model_name: facebook/blenderbot_small-90M + bot_name: hf-transformers ---- Seed Conversation:: @@ -275,12 +334,16 @@ limits: max_threads_per_user: 10 #<1> max_threads_per_topic: 3 #<2> max_turns_per_thread: 4 #<3> - reward: '0.01' #<4> + reward: '0.01' #<4> + max_human_users_per_thread: 2 #<5> + human_moderator: 'yes' #<6> ---- <1> Maximum number of threads (mtuk/assignments) a worker can do <2> Maximum number of threads (mtuk/assignments) we need for a topic (mturk/HIT) <3> Maximum number of worker replies required in a thread (/assignment) to consider it as complete <4> Reward amount. Note: currently payment can be provided to MTurk workers only; we dont have our own payment processing backend. +<5> Maximum number of human users allowed in a thread. This is useful when we want to have a human moderator in a thread, or when we want to have a group chat (e.g., two volunteers and one bot moderator). +<6> Whether to have a human moderator in a thread (only allowing `yes` or `no`). If set to `yes`, then a human moderator will be added to the thread. The human moderator will be assigned to the thread as soon as the first worker joins the thread. [#conf-flask] == Flask Server Settings @@ -298,9 +361,16 @@ Here we say that, we expose access to `flask.config` datastructure: anything we flask_config: #<5> DATABASE_FILE_NAME: 'sqlite-dev-01.db' #<1> SQLALCHEMY_TRACK_MODIFICATIONS: false + SERVER_NAME: 127.0.0.1:7072 #<2> + PREFERRED_URL_SCHEME: https #<3> ---- -<1> `DATABASE_FILE_NAME` is the filename for sqlite3 databse. - +<1> `DATABASE_FILE_NAME` is the filename for sqlite3 databse. +<2> `SERVER_NAME` affects the URL launched with MTurk HIT. + If you are running the server locally, then you may want to set `SERVER_NAME` to local IP address and port number. + If you are running the server on a public domain, then you may want to set `SERVER_NAME` to the domain name. +<3> `PREFERRED_URL_SCHEME` is the protocol (http or https) for the server. **Note**: MTurk only accepts HTTPS. + Currently, we are using a self-signed certificate for HTTPS. If you want to use a real certificate, + you might need to modify the code in `app.py:main()` Make sure to comment out SERVER_NAME and PREFERRED_URL_SCHEME to run locally. @@ -355,3 +425,60 @@ To launch HITs on mturk, follow these steps . Login as admin user (See <<#quickstart>>) . Go to _Admin Dashboard > Topics > Launch on Mturk or Mturk Sandbox_ (depending on cofig) +[#conf-customization] +== Customization Settings +This session is for any custom settings that you want to pass to the bot during the task creation. The arguments will be shown to the admin on the topics page. +[source,yaml] +---- +customization: + argument1: #<1> + - option1 + - option2 + - option3 + argument2: 'option1' #<2> +---- +<1> If there are a list of options, they will be shown as a dropdown. +<2> If there is only one option, it will be shown as a text box. Here, the default value of the text input field will be 'option1'. +After the admin clicks on the 'Create Task' button, the arguments will be passed to the bot as a dictionary (e.g., `{argument1: option1, argument2: option1}`. + +[#conf-limits_to_display] +== Limits to Display Settings +This session is for the admin to choose which "limits" to display on the topics page. +[source,yaml] +---- +limits_to_display: + - max_threads_per_user + - max_threads_per_topic + - multi_topics_creation + - multi_tasks_launch + - max_human_users_per_thread + - human_moderator + - max_turns_per_thread +---- + +Here are some pre-defined configurations options when launching (one) task(s) or for global settings. You can choose to display some of them including the followings under the `limits_to_display`. + +Here are all the options for launching (one) task(s): +If not included, the default values will be used. + + - `max_threads_per_topic`: how many threads per topic we allow - integer + default: 10 + - `max_turns_per_thread`: how many turns per thread we allow the volunteer to chat (with the bot) - integer + default: 10 + - `max_human_users_per_thread`: how many volunteers can chat in the same thread - integer + default: 1 + - `human_moderator`: whether the moderator of the thread is a human or not - two options: 'yes' or 'no' + default: 'no' + - `reward`: how much we pay the volunteer for completing the task (on MTurk) - float + default: 0.01 + +Here are all the options for global settings (shown on the top of the admin topics page): +If not included, the corresponding panel won't appear on the admin topics page. +- `max_threads_per_user`: how many threads per user we allow - integer +default: 10 +- `multi_topics_creation`: whether we allow the admin to create multiple topics at once +default: 'no' +- `multi_tasks_launch`: whether we allow the admin to launch multiple tasks at once +default: 'no' + +Basically, if you include the option in the `limits_to_display`, the corresponding panel will appear on the admin topics page. If you don't include the option, the corresponding panel won't appear on the admin topics page. diff --git a/docs/02-add-custom-bots.adoc b/docs/02-add-custom-bots.adoc index 16e9e41..bd6e2e4 100644 --- a/docs/02-add-custom-bots.adoc +++ b/docs/02-add-custom-bots.adoc @@ -5,14 +5,15 @@ include::_head.adoc[] If `\\__init__.py` file is found at the root of task directory, then the directory is treated as python module and imported it. -TIP: Refer to `example-chat-task` directory for an example. +TIP: Refer to `boteval/bots.py` file and `example-chat-task` directory for an example. -NOTE: you may have to install additional requirements/libs for your code. +NOTE: You may have to install additional requirements/libs for your code. + + To add a custom bot, you must implement a class that extends `boteval.bots.BotAgent` class. .Example bot and transform [source,python,linenums] ---- -from typing import Any +from typing import Any, Dict, List from boteval import log, C, registry as R from boteval.bots import BotAgent @@ -20,25 +21,21 @@ from boteval.transforms import BaseTransform, SpacySplitter from boteval.model import ChatMessage -@R.register(R.BOT, name="my-dummy-bot") -class MyDummpyBot(BotAgent): - - def __init__(self, **kwargs): - super().__init__(name="dummybot", **kwargs) - self.args = kwargs - log.info(f"{self.name} initialized; args={self.args}") +@R.register(R.BOT, 'dummybot') +class DummyBot(BotAgent): + + NAME = 'dummybot' - def talk(self) -> dict[str, Any]: - if not self.last_msg: - reply = f"Hello there! I am {C.BOT_DISPLAY_NAME}." - elif 'ping' in self.last_msg['text'].lower(): - reply = 'pong' - else: - reply = f"Dummy reply for -- {self.last_msg['text']}" - return dict(text=reply) + def init_chat_context(self, init_messages: List[Dict[str, Any]]): + log.debug(f'Initializing chat context with {len(init_messages)} messages') + def talk(self, **kwargs): + context = (self.last_msg or {}).get('text', '') + if context.lower() == 'ping': + return 'pong' + return dict(text="dummybot reply --" + context[-30:], data={'speaker_id': 'Moderator'}) def hear(self, msg: dict[str, Any]): - self.last_msg = msg + log.debug(f'dummybot is hearing messages') @R.register(R.TRANSFORM, name='my-transform') diff --git a/requirements.txt b/requirements.txt index 05b4f5e..a4eca44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,17 @@ -Flask==2.1 +Flask==2.1.0 ruamel.yaml>=0.17 boto3>=1.24 cachetools>=5.2 #flask-socketio #eventlet +Werkzeug==2.2.2 flask-login flask_sqlalchemy blinker>=1.5 requests transformers -openai +openai==0.28.1 spacy SQLAlchemy==1.4.46 loguru +pyopenssl==23.3.0 \ No newline at end of file