- # - #
- - - - -def find_local_files(html): - """ - constructs a list of BareFiles, so that they can later be replaced with a url to a canvas thing - """ - from bs4 import BeautifulSoup - - soup = BeautifulSoup(html,features="lxml") - - local_files = {} - - all_links = soup.findAll("a") - - if all_links: - for file in all_links: - href = file["href"] - if path.exists(path.abspath(href)): - local_files[href] = BareFile(path.abspath(href)) - - return local_files - - - -def adjust_html_for_files(html, published_files, courseid): - - - # need to write a url like this : - # Download - - - from bs4 import BeautifulSoup - - soup = BeautifulSoup(html,features="lxml") - - all_files = soup.findAll("a") - - if all_files: - for file in all_files: - href = file["href"] - if path.exists(path.abspath(href)): - # find the image in the list of published images, replace url, do more stuff. - local_file = published_files[href] - file['href'] = local_file.make_href_url(courseid) - file['class'] = "instructure_file_link instructure_scribd_file" - file['title'] = local_file.name # what it's called when you download it??? - file['data-api-endpoint'] = local_file.make_api_endpoint_url(courseid) - file['data-api-returntype'] = 'File' - - return str(soup) - - - -def get_root_folder(course): - for f in course.get_folders(): - if f.full_name == 'course files': - return f - - - - - - - -class AlreadyExists(Exception): - - def __init__(self, message, errors=""): - # Call the base class constructor with the parameters it needs - super().__init__(message) - - self.errors = errors - -class SetupError(Exception): - - def __init__(self, message, errors=""): - # Call the base class constructor with the parameters it needs - super().__init__(message) - - self.errors = errors - - - - -class DoesntExist(Exception): - """ - Used when getting a thing, but it doesn't exist - """ - - def __init__(self, message, errors=""): - # Call the base class constructor with the parameters it needs - super().__init__(message) - - self.errors = errors - - - - - - -def get_assignment_group_id(assignment_group_name, course, create_if_necessary=False): - - existing_groups = course.get_assignment_groups() - - if not isinstance(assignment_group_name,str): - raise RuntimeError(f'assignment_group_name must be a string, but I got {assignment_group_name} of type {type(assignment_group_name)}') - - - for g in existing_groups: - if g.name == assignment_group_name: - return g.id - - - - if create_if_necessary: - msg = f'making new assignment group `{assignment_group_name}`' - logging.info(msg) - - group = course.create_assignment_group(name=assignment_group_name) - group.edit(name=assignment_group_name) # this feels stupid. didn't i just request its name be this? - - return group.id - else: - raise DoesntExist(f'cannot get assignment group id because an assignment group of name {assignment_group_name} does not already exist, and `create_if_necessary` is set to False') - - - - - -def create_or_get_assignment(name, course, even_if_exists = False): - - if is_assignment_already_uploaded(name,course): - if even_if_exists: - return find_assignment_in_course(name,course) - else: - raise AlreadyExists(f"assignment {name} already exists") - else: - # make new assignment of name in course. - return course.create_assignment(assignment={'name':name}) - - - -def create_or_get_page(name, course, even_if_exists): - if is_page_already_uploaded(name,course): - - if even_if_exists: - return find_page_in_course(name,course) - else: - raise AlreadyExists(f"page {name} already exists") - else: - # make new assignment of name in course. - result = course.create_page(wiki_page={'body':"empty page",'title':name}) - return result - - - - -def create_or_get_module(module_name, course): - - try: - return get_module(module_name, course) - except DoesntExist as e: - return course.create_module(module={'name':module_name}) - - - - -def get_module(module_name, course): - """ - returns - * Module if such a module exists, - * raises if not - """ - modules = course.get_modules() - - for m in modules: - if m.name == module_name: - return m - - raise DoesntExist(f"tried to get module {module_name}, but it doesn't exist in the course") - - -def get_subfolder_named(folder, subfolder_name): - - assert '/' not in subfolder_name, "this is likely broken if subfolder has a / in its name, / gets converted to something else by Canvas. don't use / in subfolder names, that's not allowed" - - current_subfolders = folder.get_folders() - for f in current_subfolders: - if f.name == subfolder_name: - return f - - raise DoesntExist(f'a subfolder of {folder.name} named {subfolder_name} does not currently exist') - - -def delete_module(module_name, course, even_if_exists): - - if even_if_exists: - try: - m = get_module(module_name, course) - m.delete() - except DoesntExist as e: - return - - else: - # this path is expected to raise if the module doesn't exist - m = get_module(module_name, course) - m.delete() - +from markdown2canvas.free_functions import * ################## classes +from markdown2canvas.base_classes import CanvasObject, Document +from markdown2canvas.classes import Page, Assignment, Image, BareFile, Link, File -class CanvasObject(object): - """ - A base class for wrapping canvas objects. - """ - - def __init__(self,canvas_obj=None): - - super(object, self).__init__() - - self.canvas_obj = canvas_obj - - - - -class Document(CanvasObject): - """ - A base class which handles common pieces of interface for things like Pages and Assignments - - This type is abstract. Assignments and Pages both derive from this. - - At least two files are required in the folder for a Document: - - 1. `meta.json` - 2. `source.md` - - You may have additional files in the folder for a Document, such as images and files to include in the content on Canvas. This library will automatically upload those for you! - """ - - def __init__(self,folder,course=None): - """ - Construct a Document. - Reads the meta.json file and source.md files - from the specified folder. - """ - - super(Document,self).__init__(folder) - import json, os - from os.path import join - - self.folder = folder - - # try to open, and see if the meta and source files exist. - # if not, raise - self.metaname = path.join(folder,'meta.json') - with open(self.metaname,'r',encoding='utf-8') as f: - self.metadata = json.load(f) - - self.sourcename = path.join(folder,'source.md') - - - # variables populated from the metadata. should these even exist? IDK - self.name = None - self.style_path = None - self.replacements_path = None - - # populate the above variables from the meta.json file - self._set_from_metadata() - - - # these internally-used variables are used to carry state between functions - self._local_images = None - self._local_files = None - self._translated_html = None - - - def _set_from_metadata(self): - """ - this function is called during `__init__`. - """ - - self.name = self.metadata['name'] - - if 'modules' in self.metadata: - self.modules = self.metadata['modules'] - else: - self.modules = [] - - if 'indent' in self.metadata: - self.indent = self.metadata['indent'] - else: - self.indent = 0 - - if 'style' in self.metadata: - self.style_path = find_in_containing_directory_path(self.metadata['style']) - else: - self.style_path = get_default_style_name() # could be None if doesn't exist - if self.style_path: - self.style_path = find_in_containing_directory_path(self.style_path) - - if 'replacements' in self.metadata: - self.replacements_path = find_in_containing_directory_path(self.metadata['replacements']) - else: - self.replacements_path = get_default_replacements_name() # could be None if doesn't exist - if self.replacements_path: - self.replacements_path = find_in_containing_directory_path(self.replacements_path) - - - - - def translate_to_html(self,course): - """ - populates the internal variables with the results of translating from markdown to html. - - This step requires the `course` since this library allows for referencing of content already on canvas (or to be later published on Canvas) - - The main result of translation is held in self._translated_html. The local content (on YOUR computer, NOT Canvas) is parsed out and held in `self._local_images` and `self._local_files`. - - * This function does NOT make content appear on Canvas. - * It DOES leave behind a temporary file: `{folder}/styled_source.md`. Be sure to add `*/styled_source.md` to your .gitignore for your course! - """ - from os.path import join - - if self.style_path: - outname = join(self.folder,"styled_source.md") - apply_style_markdown(self.sourcename, self.style_path, outname) - - translated_html_without_hf = markdown2html(outname,course, self.replacements_path) - - self._translated_html = apply_style_html(translated_html_without_hf, self.style_path, outname) - else: - self._translated_html = markdown2html(self.sourcename,course, self.replacements_path) - - - self._local_images = find_local_images(self._translated_html) - self._local_files = find_local_files(self._translated_html) - - - - - - def publish_linked_content_and_adjust_html(self,course,overwrite=False): - """ - this function should be called *after* `translate_to_html`, since it requires the internal variables that other function populates - - the result of this function is written to `folder/result.html` - - * This function does NOT make content appear on Canvas. - * It DOES leave behind a temporary file: `{folder}/result.html`. Be sure to add `*/result.html` to your .gitignore for your course! - """ - - # first, publish the local images. - for im in self._local_images.values(): - im.publish(course,'images', overwrite=overwrite) - - for file in self._local_files.values(): - file.publish(course,'automatically_uploaded_files', overwrite=overwrite) - - - # then, deal with the urls - self._translated_html = adjust_html_for_images(self._translated_html, self._local_images, course.id) - self._translated_html = adjust_html_for_files(self._translated_html, self._local_files, course.id) - - save_location = path.join(self.folder,'result.html') - with open(save_location,'w',encoding='utf-8') as result: - result.write(self._translated_html) - - - - - - - - - - def _construct_dict_of_props(self): - """ - construct a dictionary of properties, such that it can be used to `edit` a canvas object. - """ - d = {} - return d - - - def ensure_in_modules(self, course): - """ - makes sure this item is listed in the Module on Canvas. If it's not, it's added to the bottom. There's not currently any way to control order. - - If the item doesn't already exist, this function will raise. Be sure to actually publish the content first. - """ - - if not self.canvas_obj: - raise DoesntExist(f"trying to make sure an object is in its modules, but this item ({self.name}) doesn't exist on canvas yet. publish it first.") - - for module_name in self.modules: - module = create_or_get_module(module_name, course) - - if not self.is_in_module(module_name, course): - - if self.metadata['type'] == 'page': - content_id = self.canvas_obj.page_id - elif self.metadata['type'] == 'assignment': - content_id = self.canvas_obj.id - - - module.create_module_item(module_item={'type':self.metadata['type'], 'content_id':content_id, 'indent':self.indent}) - - - def is_in_module(self, module_name, course): - """ - checks whether this content is an item in the listed module, where `module_name` is a string. It's case sensitive and exact. - - passthrough raise if the module doesn't exist - """ - - module = get_module(module_name,course) - - for item in module.get_module_items(): - - if item.type=='Page': - if self.metadata['type']=='page': - - if course.get_page(item.page_url).title == self.name: - return True - - else: - continue - - - if item.type=='Assignment': - if self.metadata['type']=='assignment': - - if course.get_assignment(assignment=item.content_id).name == self.name: - return True - else: - continue - - return False - - -class Page(Document): - """ - a Page is an abstraction around content for plain old canvas pages, which facilitates uploading to Canvas. - - folder -- a string, the name of the folder we're going to read data from. - """ - def __init__(self, folder): - super(Page, self).__init__(folder) - - - def _set_from_metadata(self): - super(Page,self)._set_from_metadata() - - - def publish(self, course, overwrite=False): - """ - if `overwrite` is False, then if an assignment is found with the same name already, the function will decline to make any edits. - - That is, if overwrite==False, then this function will only succeed if there's no existing assignment of the same name. - - This base-class function will handle things like the html, images, etc. - - Other derived-class `publish` functions will handle things like due-dates for assignments, etc. - """ - - logging.info(f'starting translate and upload process for Page `{self.name}`') - - - try: - page = create_or_get_page(self.name, course, even_if_exists=overwrite) - except AlreadyExists as e: - if not overwrite: - raise e - - self.canvas_obj = page - - self.translate_to_html(course) - - self.publish_linked_content_and_adjust_html(course, overwrite=overwrite) - - d = self._construct_dict_of_props() - page.edit(wiki_page=d) - - self.ensure_in_modules(course) - - logging.info(f'done uploading {self.name}') - - - - def _construct_dict_of_props(self): - - d = super(Page,self)._construct_dict_of_props() - - d['body'] = self._translated_html - d['title'] = self.name - - return d - - def __str__(self): - result = f"Page({self.folder})" - return result - - -class Assignment(Document): - """docstring for Assignment""" - def __init__(self, folder): - super(Assignment, self).__init__(folder) - - # self._set_from_metadata() # <-- this is called from the base __init__ - - def __str__(self): - result = f"Assignment({self.folder})" - return result - - def _get_list_of_canvas_properties_(self): - doc_url = "https://canvas.instructure.com/doc/api/assignments.html#method.assignments_api.update" - thing = "Request Parameters:" - raise NotImplementedError(f"this function is not implemented, but is intended to provide a programmatic way to determine the validity of a property name. see `{doc_url}`") - - - def _set_from_metadata(self): - super(Assignment,self)._set_from_metadata() - - default_to_none = lambda propname: self.metadata[propname] if propname in self.metadata else None - - self.allowed_extensions = default_to_none('allowed_extensions') - - self.points_possible = default_to_none('points_possible') - - self.unlock_at = default_to_none('unlock_at') - self.lock_at = default_to_none('lock_at') - self.due_at = default_to_none('due_at') - - self.published = default_to_none('published') - - self.submission_types = default_to_none('submission_types') - - self.external_tool_tag_attributes = default_to_none('external_tool_tag_attributes') - self.omit_from_final_grade = default_to_none('omit_from_final_grade') - - self.grading_type = default_to_none('grading_type') - self.assignment_group_name = default_to_none('assignment_group_name') - - self._validate_props() - - def _validate_props(self): - - - if self.allowed_extensions is not None and self.submission_types is None: - print('warning: using allowed_extensions but submission_types is not specified in the meta.json file for this assignment. you should probably use / include ["online_upload"]. valid submission_types can be found at https://canvas.instructure.com/doc/api/assignments.html#method.assignments_api.update') - - if self.allowed_extensions is not None and not isinstance(self.allowed_extensions,list): - print('warning: allowed_extensions must be a list') - - if self.submission_types is not None and not isinstance(self.submission_types,list): - print('warning: submission_types must be a list. Valid submission_types can be found at https://canvas.instructure.com/doc/api/assignments.html#method.assignments_api.update') - - if self.allowed_extensions is not None and isinstance(self.submission_types,list): - if 'online_upload' not in self.submission_types: - print('warning: using allowed_extensions, but "online_upload" is not in your list of submission_types. you should probably add it.') - - def _construct_dict_of_props(self): - - d = super(Assignment,self)._construct_dict_of_props() - d['name'] = self.name - d['description'] = self._translated_html - - if not self.allowed_extensions is None: - d['allowed_extensions'] = self.allowed_extensions - - if not self.points_possible is None: - d['points_possible'] = self.points_possible - - if not self.unlock_at is None: - d['unlock_at'] = self.unlock_at - if not self.due_at is None: - d['due_at'] = self.due_at - if not self.lock_at is None: - d['lock_at'] = self.lock_at - - if not self.published is None: - d['published'] = self.published - - if not self.submission_types is None: - d['submission_types'] = self.submission_types - - if not self.external_tool_tag_attributes is None: - d['external_tool_tag_attributes'] = self.external_tool_tag_attributes - - if not self.omit_from_final_grade is None: - d['omit_from_final_grade'] = self.omit_from_final_grade - - if not self.grading_type is None: - d['grading_type'] = self.grading_type - - return d - - - - - def ensure_in_assignment_groups(self, course, create_if_necessary=False): - - if self.assignment_group_name is None: - logging.info(f'when putting assignment {self.name} into group, taking no action because no assignment group specified') - return - - assignment_group_id = get_assignment_group_id(self.assignment_group_name, course, create_if_necessary) # todo: change this to try/except, instead of passing `create_if_necessary` to the get function. getting gets. it shouldn't create. - self.canvas_obj.edit(assignment={'assignment_group_id':assignment_group_id}) - - - - def publish(self, course, overwrite=False, create_modules_if_necessary=False, create_assignment_group_if_necessary=False): - """ - if `overwrite` is False, then if an assignment is found with the same name already, the function will decline to make any edits. - - That is, if overwrite==False, then this function will only succeed if there's no existing assignment of the same name. - """ - - logging.info(f'starting translate and upload process for Assignment `{self.name}`') - - - # need a remote object to work with - assignment = None - try: - assignment = create_or_get_assignment(self.name, course, overwrite) - except AlreadyExists as e: - if not overwrite: - raise e - - self.canvas_obj = assignment - - self.translate_to_html(course) - - self.publish_linked_content_and_adjust_html(course, overwrite=overwrite) - - # now that we have the assignment, we'll update its content. - - new_props=self._construct_dict_of_props() - - # for example, - # ass[0].edit(assignment={'lock_at':datetime.datetime(2021, 8, 17, 4, 59, 59),'due_at':datetime.datetime(2021, 8, 17, 4, 59, 59)}) - # we construct the dict of values in the _construct_dict_of_props() function. - - assignment.edit(assignment=new_props) - - self.ensure_in_modules(course) - self.ensure_in_assignment_groups(course,create_if_necessary=create_assignment_group_if_necessary) - - logging.info(f'done uploading {self.name} to Canvas') - - return True - - - - - - - -class Image(CanvasObject): - """ - A wrapper class for images on Canvas - """ - - - def __init__(self, filename, alttext = ''): - super(Image, self).__init__() - - self.givenpath = filename - self.filename = filename - # self.name = path.basename(filename) - # self.folder = path.abspath(filename) - - self.name = path.split(filename)[1] - self.folder = path.split(filename)[0] - - self.alttext = alttext - - - #- # - #
- - def publish(self, course, dest, overwrite=False, raise_if_already_uploaded = False): - """ - - - see also https://canvas.instructure.com/doc/api/file.file_uploads.html - """ - - if overwrite: - on_duplicate = 'overwrite' - else: - on_duplicate = 'rename' - - - # this still needs to be adjusted to capture the Canvas image, in case it exists - if overwrite: - logging.debug('uploading {} to {}'.format(self.givenpath, dest)) - success_code, json_response = course.upload(self.givenpath, parent_folder_path=dest,on_duplicate=on_duplicate) - logging.debug('success_code from uploading was {}'.format(success_code)) - logging.debug('json response from uploading was {}'.format(json_response)) - - if not success_code: - print(f'failed to upload... {self.givenpath}') - - self.canvas_obj = course.get_file(json_response['id']) - return self.canvas_obj - - else: - if is_file_already_uploaded(self.givenpath,course): - if raise_if_already_uploaded: - raise AlreadyExists(f'image {self.name} already exists in course {course.name}, but you don\'t want to overwrite.') - else: - img_on_canvas = find_file_in_course(self.givenpath,course) - else: - # get the remote image - print(f'file not already uploaded, uploading {self.name}') - - success_code, json_response = course.upload(self.givenpath, parent_folder_path=dest,on_duplicate=on_duplicate) - img_on_canvas = course.get_file(json_response['id']) - if not success_code: - print(f'failed to upload... {self.givenpath}') - - - self.canvas_obj = img_on_canvas - - return img_on_canvas - - def make_src_url(self,courseid): - """ - constructs a string which can be used to embed the image in a Canvas page. - - sadly, the JSON back from Canvas doesn't just produce this for us. lame. - - """ - import canvasapi - im = self.canvas_obj - assert(isinstance(self.canvas_obj, canvasapi.file.File)) - - n = im.url.find('/files') - - url = im.url[:n]+'/courses/'+str(courseid)+'/files/'+str(im.id)+'/preview' - - return url - - def make_api_endpoint_url(self,courseid): - import canvasapi - im = self.canvas_obj - assert(isinstance(self.canvas_obj, canvasapi.file.File)) - - n = im.url.find('/files') - - url = im.url[:n] + '/api/v1/courses/' + str(courseid) + '/files/' + str(im.id) - return url - # data-api-endpoint="https://uws-td.instructure.com/api/v1/courses/3099/files/219835" - - - def __str__(self): - result = "\n" - result = result + f'givenpath: {self.givenpath}\n' - result = result + f'name: {self.name}\n' - result = result + f'folder: {self.folder}\n' - result = result + f'alttext: {self.alttext}\n' - result = result + f'canvas_obj: {self.canvas_obj}\n' - url = self.make_src_url('fakecoursenumber') - result = result + f'constructed canvas url: {url}\n' - - return result+'\n' - - def __repr__(self): - return str(self) - - - - -class BareFile(CanvasObject): - """ - A wrapper class for bare, unwrapped files on Canvas, for link to inline. - """ - - - def __init__(self, filename): - super(BareFile, self).__init__() - - self.givenpath = filename - self.filename = filename - self.name = path.basename(filename) - self.folder = path.abspath(filename) - - # self.name = path.split(filename)[1] - # self.folder = path.split(filename)[0] - - - - def publish(self, course, dest, overwrite=False, raise_if_already_uploaded = False): - """ - - - see also https://canvas.instructure.com/doc/api/file.file_uploads.html - """ - - if overwrite: - on_duplicate = 'overwrite' - else: - on_duplicate = 'rename' - - - - # this still needs to be adjusted to capture the Canvas file, in case it exists - if overwrite: - success_code, json_response = course.upload(self.givenpath, parent_folder_path=dest,on_duplicate=on_duplicate) - if not success_code: - print(f'failed to upload... {self.givenpath}') - else: - print(f'overwrote {self.name}') - - self.canvas_obj = course.get_file(json_response['id']) - return self.canvas_obj - - else: - if is_file_already_uploaded(self.givenpath,course): - if raise_if_already_uploaded: - raise AlreadyExists(f'file {self.name} already exists in course {course.name}, but you don\'t want to overwrite.') - else: - file_on_canvas = find_file_in_course(self.givenpath,course) - else: - # get the remote file - print(f'file not already uploaded, uploading {self.name}') - - success_code, json_response = course.upload(self.givenpath, parent_folder_path=dest,on_duplicate=on_duplicate) - file_on_canvas = course.get_file(json_response['id']) - if not success_code: - print(f'failed to upload... {self.givenpath}') - - - self.canvas_obj = file_on_canvas - - return file_on_canvas - - - - - def make_href_url(self,courseid): - """ - constructs a string which can be used to reference the file in a Canvas page. - - sadly, the JSON back from Canvas doesn't just produce this for us. lame. - - """ - import canvasapi - file = self.canvas_obj - assert(isinstance(self.canvas_obj, canvasapi.file.File)) - - n = file.url.find('/files') - - url = file.url[:n]+'/courses/'+str(courseid)+'/files/'+str(file.id)+'/download?wrap=1' - - return url - - - def make_api_endpoint_url(self,courseid): - import canvasapi - file = self.canvas_obj - assert(isinstance(self.canvas_obj, canvasapi.file.File)) - - n = file.url.find('/files') - - url = file.url[:n] + '/api/v1/courses/' + str(courseid) + '/files/' + str(file.id) - return url - # data-api-endpoint="https://uws-td.instructure.com/api/v1/courses/3099/files/219835" - - - def __str__(self): - result = "\n" - result = result + f'givenpath: {self.givenpath}\n' - result = result + f'name: {self.name}\n' - result = result + f'folder: {self.folder}\n' - result = result + f'alttext: {self.alttext}\n' - result = result + f'canvas_obj: {self.canvas_obj}\n' - url = self.make_href_url('fakecoursenumber') - result = result + f'constructed canvas url: {url}\n' - - return result+'\n' - - def __repr__(self): - return str(self) - - - - - - - - - - - - - - - - - - - - - -class Link(CanvasObject): - """ - a containerization of url's, for uploading to Canvas modules - """ - def __init__(self, folder): - super(Link, self).__init__() - self.folder = folder - - import json, os - from os.path import join - - self.metaname = path.join(folder,'meta.json') - with open(self.metaname,'r',encoding='utf-8') as f: - self.metadata = json.load(f) - - if 'indent' in self.metadata: - self.indent = self.metadata['indent'] - else: - self.indent = 0 - - def __str__(self): - result = f"Link({self.metadata['external_url']})" - return result - - def __repr__(self): - return str(self) - - - def publish(self, course, overwrite=False): - - for m in self.metadata['modules']: - if link_on_canvas:= self.is_in_module(course, m): - if not overwrite: - n = self.metadata['external_url'] - raise AlreadyExists(f'trying to upload {self}, but is already on Canvas in module {m}') - else: - link_on_canvas.edit(module_item={'external_url':self.metadata['external_url'],'title':self.metadata['name'], 'new_tab':bool(self.metadata['new_tab'])}) - - else: - mod = create_or_get_module(m, course) - mod.create_module_item(module_item={'type':'ExternalUrl','external_url':self.metadata['external_url'],'title':self.metadata['name'], 'new_tab':bool(self.metadata['new_tab']), 'indent':self.indent}) - - - def is_already_uploaded(self, course): - for m in self.metadata['modules']: - if not self.is_in_module(course, m): - return False - - return True - - - - def is_in_module(self, course, module_name): - try: - module = get_module(module_name,course) - except DoesntExist as e: - return None - - - for item in module.get_module_items(): - - if item.type=='ExternalUrl' and item.external_url==self.metadata['external_url']: - return item - else: - continue - - return None - - - - -class File(CanvasObject): - """ - a containerization of arbitrary files, for uploading to Canvas - """ - def __init__(self, folder): - super(File, self).__init__(folder) - - import json, os - from os.path import join - - self.folder = folder - - self.metaname = path.join(folder,'meta.json') - with open(self.metaname,'r',encoding='utf-8') as f: - self.metadata = json.load(f) - - try: - self.title = self.metadata['title'] - except: - self.title = self.metadata['filename'] - - - if 'indent' in self.metadata: - self.indent = self.metadata['indent'] - else: - self.indent = 0 - - - def __str__(self): - result = f"File({self.metadata})" - return result - - def __repr__(self): - return str(self) - - - def _upload_(self, course): - pass - - - def publish(self, course, overwrite=False): - """ - publishes a file to Canvas in a particular folder - """ - - on_duplicate='overwrite' - if (file_on_canvas:= self.is_already_uploaded(course)) and not overwrite: - # on_duplicate='rename' - n = self.metadata['filename'] - # content_id = file_on_canvas.id - - raise AlreadyExists(f'The file {n} is already on Canvas and `not overwrite`.') - else: - root = get_root_folder(course) - - d = self.metadata['destination'] - d = d.split('/') - - curr_dir = root - for subd in d: - try: - curr_dir = get_subfolder_named(curr_dir, subd) - except DoesntExist as e: - curr_dir = curr_dir.create_folder(subd) - - filepath_to_upload = path.join(self.folder,self.metadata['filename']) - reply = curr_dir.upload(file=filepath_to_upload,on_duplicate=on_duplicate) - - if not reply[0]: - raise RuntimeError(f'something went wrong uploading {filepath_to_upload}') - - file_on_canvas = reply[1] - content_id = file_on_canvas['id'] - - - # now to make sure it's in the right modules - for module_name in self.metadata['modules']: - module = create_or_get_module(module_name, course) - - items = module.get_module_items() - is_in = False - for item in items: - if item.type=='File' and item.content_id==content_id: - is_in = True - break - - if not is_in: - module.create_module_item(module_item={'type':'File', 'content_id':content_id, 'title':self.title, 'indent':self.indent}) - # if the title doesn't match, update it - elif item.title != self.title: - item.edit(module_item={'type':'File', 'content_id':content_id, 'title':self.title},module=module) - - - def is_in_module(self, course, module_name): - file_on_canvas = self.is_already_uploaded(course) - - if not file_on_canvas: - return False - - module = get_module(module_name,course) - - for item in module.get_module_items(): - - if item.type=='File' and item.content_id==file_on_canvas.id: - return True - else: - continue - - return False - - - def is_already_uploaded(self,course, require_same_path=True): - files = course.get_files() - - for f in files: - if f.filename == self.metadata['filename']: - - if not require_same_path: - return f - else: - containing_folder = course.get_folder(f.folder_id) - if containing_folder.full_name.startswith('course files') and containing_folder.full_name.endswith(self.metadata['destination']): - return f - - - return None - - - - - - -def page2markdown(destination, page, even_if_exists=False): - """ - takes a Page from Canvas, and saves it to a folder inside `destination` - into a markdown2canvas compatible format. - - the folder is automatically named, at your own peril. - """ - - import os - - assert(isinstance(page,canvasapi.page.Page)) - - if (path.exists(destination)) and not path.isdir(destination): - raise AlreadyExists(f'you want to save a page into directory {destination}, but it exists and is not a directory') - - - - - r = page.show_latest_revision() - body = r.body # this is the content of the page, in html. - title = r.title - - dir_name = title.replace(":","").replace(" ","_") - destdir = path.join(destination,dir_name) - if (not even_if_exists) and path.exists(destdir): - raise AlreadyExists(f'trying to save page {title} to folder {destdir}, but that already exists. If you want to force, use `even_if_exists=True`.') - - if not path.exists(destdir): - os.makedirs(destdir) - - logging.info(f'downloading page {title}, saving to folder {destdir}') - - with open(path.join(destdir,'source.md'),'w',encoding='utf-8') as file: - file.write(body) - - - d = {} - - d['name'] = title - d['type'] = 'page' - with open(path.join(destdir,'meta.json'),'w',encoding='utf-8') as file: - import json - json.dump(d, file) - - - - -def download_pages(destination, course, even_if_exists=False, name_filter=None): - """ - downloads the regular pages from a course, saving them - into a markdown2canvas compatible format. that is, as - a folder with markdown source and json metadata. - """ - - if name_filter is None: - name_filter = lambda x: True - - logging.info(f'downloading all pages from course {course.name}, saving to folder {destination}') - pages = course.get_pages() - for p in pages: - if name_filter(p.show_latest_revision().title): - page2markdown(destination,p,even_if_exists) - - -def assignment2markdown(destination, assignment, even_if_exists=False): - """ - takes a Page from Canvas, and saves it to a folder inside `destination` - into a markdown2canvas compatible format. - - the folder is automatically named, at your own peril. - """ - - import os - - assert(isinstance(assignment,canvasapi.assignment.Assignment)) - - if (path.exists(destination)) and not path.isdir(destination): - raise AlreadyExists(f'you want to save a page into directory {destination}, but it exists and is not a directory') - - - - - body = assignment.description # this is the content of the page, in html. - title = assignment.name - - destdir = path.join(destination,title) - if (not even_if_exists) and path.exists(destdir): - raise AlreadyExists(f'trying to save page {title} to folder {destdir}, but that already exists. If you want to force, use `even_if_exists=True`.') - - if not path.exists(destdir): - os.makedirs(destdir) - - logging.info(f'downloading page {title}, saving to folder {destdir}') - - with open(path.join(destdir,'source.md'),'w',encoding='utf-8') as file: - file.write(body) - - - d = {} - - d['name'] = title - d['type'] = 'assignment' - with open(path.join(destdir,'meta.json'),'w',encoding='utf-8') as file: - import json - json.dump(d, file) - -def download_assignments(destination, course, even_if_exists=False, name_filter=None): - """ - downloads the regular pages from a course, saving them - into a markdown2canvas compatible format. that is, as - a folder with markdown source and json metadata. - """ - - if name_filter is None: - name_filter = lambda x: True - logging.info(f'downloading all pages from course {course.name}, saving to folder {destination}') - assignments = course.get_assignments() - for a in assignments: - if name_filter(a.name): - assignment2markdown(destination,a,even_if_exists) diff --git a/markdown2canvas/base_classes/__init__.py b/markdown2canvas/base_classes/__init__.py new file mode 100644 index 0000000..cc04458 --- /dev/null +++ b/markdown2canvas/base_classes/__init__.py @@ -0,0 +1,235 @@ + +import os.path as path +import os + +import canvasapi +from markdown2canvas.free_functions import * + +class CanvasObject(object): + """ + A base class for wrapping canvas objects. + """ + + def __init__(self,canvas_obj=None): + + super(object, self).__init__() + + self.canvas_obj = canvas_obj + + + + +class Document(CanvasObject): + """ + A base class which handles common pieces of interface for things like Pages and Assignments + + This type is abstract. Assignments and Pages both derive from this. + + At least two files are required in the folder for a Document: + + 1. `meta.json` + 2. `source.md` + + You may have additional files in the folder for a Document, such as images and files to include in the content on Canvas. This library will automatically upload those for you! + """ + + def __init__(self,folder,course=None): + """ + Construct a Document. + Reads the meta.json file and source.md files + from the specified folder. + """ + + super(Document,self).__init__(folder) + import json, os + from os.path import join + + self.folder = folder + + # try to open, and see if the meta and source files exist. + # if not, raise + self.metaname = path.join(folder,'meta.json') + with open(self.metaname,'r',encoding='utf-8') as f: + self.metadata = json.load(f) + + self.sourcename = path.join(folder,'source.md') + + + # variables populated from the metadata. should these even exist? IDK + self.name = None + self.style_path = None + self.replacements_path = None + + # populate the above variables from the meta.json file + self._set_from_metadata() + + + # these internally-used variables are used to carry state between functions + self._local_images = None + self._local_files = None + self._translated_html = None + + + def _set_from_metadata(self): + """ + this function is called during `__init__`. + """ + + self.name = self.metadata['name'] + + if 'modules' in self.metadata: + self.modules = self.metadata['modules'] + else: + self.modules = [] + + if 'indent' in self.metadata: + self.indent = self.metadata['indent'] + else: + self.indent = 0 + + if 'style' in self.metadata: + self.style_path = find_in_containing_directory_path(self.metadata['style']) + else: + self.style_path = get_default_style_name() # could be None if doesn't exist + if self.style_path: + self.style_path = find_in_containing_directory_path(self.style_path) + + if 'replacements' in self.metadata: + self.replacements_path = find_in_containing_directory_path(self.metadata['replacements']) + else: + self.replacements_path = get_default_replacements_name() # could be None if doesn't exist + if self.replacements_path: + self.replacements_path = find_in_containing_directory_path(self.replacements_path) + + + + + def translate_to_html(self,course): + """ + populates the internal variables with the results of translating from markdown to html. + + This step requires the `course` since this library allows for referencing of content already on canvas (or to be later published on Canvas) + + The main result of translation is held in self._translated_html. The local content (on YOUR computer, NOT Canvas) is parsed out and held in `self._local_images` and `self._local_files`. + + * This function does NOT make content appear on Canvas. + * It DOES leave behind a temporary file: `{folder}/styled_source.md`. Be sure to add `*/styled_source.md` to your .gitignore for your course! + """ + from os.path import join + + if self.style_path: + outname = join(self.folder,"styled_source.md") + apply_style_markdown(self.sourcename, self.style_path, outname) + + translated_html_without_hf = markdown2html(outname,course, self.replacements_path) + + self._translated_html = apply_style_html(translated_html_without_hf, self.style_path, outname) + else: + self._translated_html = markdown2html(self.sourcename,course, self.replacements_path) + + + self._local_images = find_local_images(self._translated_html) + self._local_files = find_local_files(self._translated_html) + + + + + + def publish_linked_content_and_adjust_html(self,course,overwrite=False): + """ + this function should be called *after* `translate_to_html`, since it requires the internal variables that other function populates + + the result of this function is written to `folder/result.html` + + * This function does NOT make content appear on Canvas. + * It DOES leave behind a temporary file: `{folder}/result.html`. Be sure to add `*/result.html` to your .gitignore for your course! + """ + + # first, publish the local images. + for im in self._local_images.values(): + im.publish(course,'images', overwrite=overwrite) + + for file in self._local_files.values(): + file.publish(course,'automatically_uploaded_files', overwrite=overwrite) + + + # then, deal with the urls + self._translated_html = adjust_html_for_images(self._translated_html, self._local_images, course.id) + self._translated_html = adjust_html_for_files(self._translated_html, self._local_files, course.id) + + save_location = path.join(self.folder,'result.html') + with open(save_location,'w',encoding='utf-8') as result: + result.write(self._translated_html) + + + + + + + + + + def _construct_dict_of_props(self): + """ + construct a dictionary of properties, such that it can be used to `edit` a canvas object. + """ + d = {} + return d + + + def ensure_in_modules(self, course): + """ + makes sure this item is listed in the Module on Canvas. If it's not, it's added to the bottom. There's not currently any way to control order. + + If the item doesn't already exist, this function will raise. Be sure to actually publish the content first. + """ + + if not self.canvas_obj: + raise DoesntExist(f"trying to make sure an object is in its modules, but this item ({self.name}) doesn't exist on canvas yet. publish it first.") + + for module_name in self.modules: + module = create_or_get_module(module_name, course) + + if not self.is_in_module(module_name, course): + + if self.metadata['type'] == 'page': + content_id = self.canvas_obj.page_id + elif self.metadata['type'] == 'assignment': + content_id = self.canvas_obj.id + + + module.create_module_item(module_item={'type':self.metadata['type'], 'content_id':content_id, 'indent':self.indent}) + + + def is_in_module(self, module_name, course): + """ + checks whether this content is an item in the listed module, where `module_name` is a string. It's case sensitive and exact. + + passthrough raise if the module doesn't exist + """ + + module = get_module(module_name,course) + + for item in module.get_module_items(): + + if item.type=='Page': + if self.metadata['type']=='page': + + if course.get_page(item.page_url).title == self.name: + return True + + else: + continue + + + if item.type=='Assignment': + if self.metadata['type']=='assignment': + + if course.get_assignment(assignment=item.content_id).name == self.name: + return True + else: + continue + + return False + + diff --git a/markdown2canvas/canvas2markdown/__init__.py b/markdown2canvas/canvas2markdown/__init__.py new file mode 100644 index 0000000..d3e6131 --- /dev/null +++ b/markdown2canvas/canvas2markdown/__init__.py @@ -0,0 +1,123 @@ +import canvasapi + +def page2markdown(destination, page, even_if_exists=False): + """ + takes a Page from Canvas, and saves it to a folder inside `destination` + into a markdown2canvas compatible format. + + the folder is automatically named, at your own peril. + """ + + import os + + assert(isinstance(page,canvasapi.page.Page)) + + if (path.exists(destination)) and not path.isdir(destination): + raise AlreadyExists(f'you want to save a page into directory {destination}, but it exists and is not a directory') + + + + + r = page.show_latest_revision() + body = r.body # this is the content of the page, in html. + title = r.title + + dir_name = title.replace(":","").replace(" ","_") + destdir = path.join(destination,dir_name) + if (not even_if_exists) and path.exists(destdir): + raise AlreadyExists(f'trying to save page {title} to folder {destdir}, but that already exists. If you want to force, use `even_if_exists=True`.') + + if not path.exists(destdir): + os.makedirs(destdir) + + logger.info(f'downloading page {title}, saving to folder {destdir}') + + with open(path.join(destdir,'source.md'),'w',encoding='utf-8') as file: + file.write(body) + + + d = {} + + d['name'] = title + d['type'] = 'page' + with open(path.join(destdir,'meta.json'),'w',encoding='utf-8') as file: + import json + json.dump(d, file) + + + + +def download_pages(destination, course, even_if_exists=False, name_filter=None): + """ + downloads the regular pages from a course, saving them + into a markdown2canvas compatible format. that is, as + a folder with markdown source and json metadata. + """ + + if name_filter is None: + name_filter = lambda x: True + + logger.info(f'downloading all pages from course {course.name}, saving to folder {destination}') + pages = course.get_pages() + for p in pages: + if name_filter(p.show_latest_revision().title): + page2markdown(destination,p,even_if_exists) + + +def assignment2markdown(destination, assignment, even_if_exists=False): + """ + takes a Page from Canvas, and saves it to a folder inside `destination` + into a markdown2canvas compatible format. + + the folder is automatically named, at your own peril. + """ + + import os + + assert(isinstance(assignment,canvasapi.assignment.Assignment)) + + if (path.exists(destination)) and not path.isdir(destination): + raise AlreadyExists(f'you want to save a page into directory {destination}, but it exists and is not a directory') + + + + + body = assignment.description # this is the content of the page, in html. + title = assignment.name + + destdir = path.join(destination,title) + if (not even_if_exists) and path.exists(destdir): + raise AlreadyExists(f'trying to save page {title} to folder {destdir}, but that already exists. If you want to force, use `even_if_exists=True`.') + + if not path.exists(destdir): + os.makedirs(destdir) + + logger.info(f'downloading page {title}, saving to folder {destdir}') + + with open(path.join(destdir,'source.md'),'w',encoding='utf-8') as file: + file.write(body) + + + d = {} + + d['name'] = title + d['type'] = 'assignment' + with open(path.join(destdir,'meta.json'),'w',encoding='utf-8') as file: + import json + json.dump(d, file) + +def download_assignments(destination, course, even_if_exists=False, name_filter=None): + """ + downloads the regular pages from a course, saving them + into a markdown2canvas compatible format. that is, as + a folder with markdown source and json metadata. + """ + + if name_filter is None: + name_filter = lambda x: True + + logger.info(f'downloading all pages from course {course.name}, saving to folder {destination}') + assignments = course.get_assignments() + for a in assignments: + if name_filter(a.name): + assignment2markdown(destination,a,even_if_exists) diff --git a/markdown2canvas/classes/__init__.py b/markdown2canvas/classes/__init__.py new file mode 100644 index 0000000..99f8371 --- /dev/null +++ b/markdown2canvas/classes/__init__.py @@ -0,0 +1,653 @@ + +import canvasapi +import os.path as path +import os + +from markdown2canvas.base_classes import Document, CanvasObject +from markdown2canvas.free_functions import * + +class Page(Document): + """ + a Page is an abstraction around content for plain old canvas pages, which facilitates uploading to Canvas. + + folder -- a string, the name of the folder we're going to read data from. + """ + def __init__(self, folder): + super(Page, self).__init__(folder) + + + def _set_from_metadata(self): + super(Page,self)._set_from_metadata() + + + def publish(self, course, overwrite=False): + """ + if `overwrite` is False, then if an assignment is found with the same name already, the function will decline to make any edits. + + That is, if overwrite==False, then this function will only succeed if there's no existing assignment of the same name. + + This base-class function will handle things like the html, images, etc. + + Other derived-class `publish` functions will handle things like due-dates for assignments, etc. + """ + + logger.info(f'starting translate and upload process for Page `{self.name}`') + + + try: + page = create_or_get_page(self.name, course, even_if_exists=overwrite) + except AlreadyExists as e: + if not overwrite: + raise e + + self.canvas_obj = page + + self.translate_to_html(course) + + self.publish_linked_content_and_adjust_html(course, overwrite=overwrite) + + d = self._construct_dict_of_props() + page.edit(wiki_page=d) + + self.ensure_in_modules(course) + + logger.info(f'done uploading {self.name}') + + + + def _construct_dict_of_props(self): + + d = super(Page,self)._construct_dict_of_props() + + d['body'] = self._translated_html + d['title'] = self.name + + return d + + def __str__(self): + result = f"Page({self.folder})" + return result + + + +class Assignment(Document): + """docstring for Assignment""" + def __init__(self, folder): + super(Assignment, self).__init__(folder) + + # self._set_from_metadata() # <-- this is called from the base __init__ + + def __str__(self): + result = f"Assignment({self.folder})" + return result + + def _get_list_of_canvas_properties_(self): + doc_url = "https://canvas.instructure.com/doc/api/assignments.html#method.assignments_api.update" + thing = "Request Parameters:" + raise NotImplementedError(f"this function is not implemented, but is intended to provide a programmatic way to determine the validity of a property name. see `{doc_url}`") + + + def _set_from_metadata(self): + super(Assignment,self)._set_from_metadata() + + default_to_none = lambda propname: self.metadata[propname] if propname in self.metadata else None + + self.allowed_extensions = default_to_none('allowed_extensions') + + self.points_possible = default_to_none('points_possible') + + self.unlock_at = default_to_none('unlock_at') + self.lock_at = default_to_none('lock_at') + self.due_at = default_to_none('due_at') + + self.published = default_to_none('published') + + self.submission_types = default_to_none('submission_types') + + self.external_tool_tag_attributes = default_to_none('external_tool_tag_attributes') + self.omit_from_final_grade = default_to_none('omit_from_final_grade') + + self.grading_type = default_to_none('grading_type') + self.assignment_group_name = default_to_none('assignment_group_name') + + self._validate_props() + + def _validate_props(self): + + + if self.allowed_extensions is not None and self.submission_types is None: + print('warning: using allowed_extensions but submission_types is not specified in the meta.json file for this assignment. you should probably use / include ["online_upload"]. valid submission_types can be found at https://canvas.instructure.com/doc/api/assignments.html#method.assignments_api.update') + + if self.allowed_extensions is not None and not isinstance(self.allowed_extensions,list): + print('warning: allowed_extensions must be a list') + + if self.submission_types is not None and not isinstance(self.submission_types,list): + print('warning: submission_types must be a list. Valid submission_types can be found at https://canvas.instructure.com/doc/api/assignments.html#method.assignments_api.update') + + if self.allowed_extensions is not None and isinstance(self.submission_types,list): + if 'online_upload' not in self.submission_types: + print('warning: using allowed_extensions, but "online_upload" is not in your list of submission_types. you should probably add it.') + + def _construct_dict_of_props(self): + + d = super(Assignment,self)._construct_dict_of_props() + d['name'] = self.name + d['description'] = self._translated_html + + if not self.allowed_extensions is None: + d['allowed_extensions'] = self.allowed_extensions + + if not self.points_possible is None: + d['points_possible'] = self.points_possible + + if not self.unlock_at is None: + d['unlock_at'] = self.unlock_at + if not self.due_at is None: + d['due_at'] = self.due_at + if not self.lock_at is None: + d['lock_at'] = self.lock_at + + if not self.published is None: + d['published'] = self.published + + if not self.submission_types is None: + d['submission_types'] = self.submission_types + + if not self.external_tool_tag_attributes is None: + d['external_tool_tag_attributes'] = self.external_tool_tag_attributes + + if not self.omit_from_final_grade is None: + d['omit_from_final_grade'] = self.omit_from_final_grade + + if not self.grading_type is None: + d['grading_type'] = self.grading_type + + return d + + + + + def ensure_in_assignment_groups(self, course, create_if_necessary=False): + + if self.assignment_group_name is None: + logger.info(f'when putting assignment {self.name} into group, taking no action because no assignment group specified') + return + + assignment_group_id = get_assignment_group_id(self.assignment_group_name, course, create_if_necessary) # todo: change this to try/except, instead of passing `create_if_necessary` to the get function. getting gets. it shouldn't create. + self.canvas_obj.edit(assignment={'assignment_group_id':assignment_group_id}) + + + + def publish(self, course, overwrite=False, create_modules_if_necessary=False, create_assignment_group_if_necessary=False): + """ + if `overwrite` is False, then if an assignment is found with the same name already, the function will decline to make any edits. + + That is, if overwrite==False, then this function will only succeed if there's no existing assignment of the same name. + """ + + logger.info(f'starting translate and upload process for Assignment `{self.name}`') + + + # need a remote object to work with + assignment = None + try: + assignment = create_or_get_assignment(self.name, course, overwrite) + except AlreadyExists as e: + if not overwrite: + raise e + + self.canvas_obj = assignment + + self.translate_to_html(course) + + self.publish_linked_content_and_adjust_html(course, overwrite=overwrite) + + # now that we have the assignment, we'll update its content. + + new_props=self._construct_dict_of_props() + + # for example, + # ass[0].edit(assignment={'lock_at':datetime.datetime(2021, 8, 17, 4, 59, 59),'due_at':datetime.datetime(2021, 8, 17, 4, 59, 59)}) + # we construct the dict of values in the _construct_dict_of_props() function. + + assignment.edit(assignment=new_props) + + self.ensure_in_modules(course) + self.ensure_in_assignment_groups(course,create_if_necessary=create_assignment_group_if_necessary) + + logger.info(f'done uploading {self.name} to Canvas') + + return True + + + +class Image(CanvasObject): + """ + A wrapper class for images on Canvas + """ + + + def __init__(self, filename, alttext = ''): + super(Image, self).__init__() + + self.givenpath = filename + self.filename = filename + # self.name = path.basename(filename) + # self.folder = path.abspath(filename) + + self.name = path.split(filename)[1] + self.folder = path.split(filename)[0] + + self.alttext = alttext + + + #+ # + #
+ + def publish(self, course, dest, overwrite=False, raise_if_already_uploaded = False): + """ + + + see also https://canvas.instructure.com/doc/api/file.file_uploads.html + """ + + if overwrite: + on_duplicate = 'overwrite' + else: + on_duplicate = 'rename' + + + # this still needs to be adjusted to capture the Canvas image, in case it exists + if overwrite: + logger.debug('uploading {} to {}'.format(self.givenpath, dest)) + success_code, json_response = course.upload(self.givenpath, parent_folder_path=dest,on_duplicate=on_duplicate) + logger.debug('success_code from uploading was {}'.format(success_code)) + logger.debug('json response from uploading was {}'.format(json_response)) + + if not success_code: + print(f'failed to upload... {self.givenpath}') + + self.canvas_obj = course.get_file(json_response['id']) + return self.canvas_obj + + else: + if is_file_already_uploaded(self.givenpath,course): + if raise_if_already_uploaded: + raise AlreadyExists(f'image {self.name} already exists in course {course.name}, but you don\'t want to overwrite.') + else: + img_on_canvas = find_file_in_course(self.givenpath,course) + else: + # get the remote image + print(f'file not already uploaded, uploading {self.name}') + + success_code, json_response = course.upload(self.givenpath, parent_folder_path=dest,on_duplicate=on_duplicate) + img_on_canvas = course.get_file(json_response['id']) + if not success_code: + print(f'failed to upload... {self.givenpath}') + + + self.canvas_obj = img_on_canvas + + return img_on_canvas + + def make_src_url(self,courseid): + """ + constructs a string which can be used to embed the image in a Canvas page. + + sadly, the JSON back from Canvas doesn't just produce this for us. lame. + + """ + import canvasapi + im = self.canvas_obj + assert(isinstance(self.canvas_obj, canvasapi.file.File)) + + n = im.url.find('/files') + + url = im.url[:n]+'/courses/'+str(courseid)+'/files/'+str(im.id)+'/preview' + + return url + + def make_api_endpoint_url(self,courseid): + import canvasapi + im = self.canvas_obj + assert(isinstance(self.canvas_obj, canvasapi.file.File)) + + n = im.url.find('/files') + + url = im.url[:n] + '/api/v1/courses/' + str(courseid) + '/files/' + str(im.id) + return url + # data-api-endpoint="https://uws-td.instructure.com/api/v1/courses/3099/files/219835" + + + def __str__(self): + result = "\n" + result = result + f'givenpath: {self.givenpath}\n' + result = result + f'name: {self.name}\n' + result = result + f'folder: {self.folder}\n' + result = result + f'alttext: {self.alttext}\n' + result = result + f'canvas_obj: {self.canvas_obj}\n' + url = self.make_src_url('fakecoursenumber') + result = result + f'constructed canvas url: {url}\n' + + return result+'\n' + + def __repr__(self): + return str(self) + + + + +class BareFile(CanvasObject): + """ + A wrapper class for bare, unwrapped files on Canvas, for link to inline. + """ + + + def __init__(self, filename): + super(BareFile, self).__init__() + + self.givenpath = filename + self.filename = filename + self.name = path.basename(filename) + self.folder = path.abspath(filename) + + # self.name = path.split(filename)[1] + # self.folder = path.split(filename)[0] + + + + def publish(self, course, dest, overwrite=False, raise_if_already_uploaded = False): + """ + + + see also https://canvas.instructure.com/doc/api/file.file_uploads.html + """ + + if overwrite: + on_duplicate = 'overwrite' + else: + on_duplicate = 'rename' + + + + # this still needs to be adjusted to capture the Canvas file, in case it exists + if overwrite: + success_code, json_response = course.upload(self.givenpath, parent_folder_path=dest,on_duplicate=on_duplicate) + if not success_code: + print(f'failed to upload... {self.givenpath}') + else: + print(f'overwrote {self.name}') + + self.canvas_obj = course.get_file(json_response['id']) + return self.canvas_obj + + else: + if is_file_already_uploaded(self.givenpath,course): + if raise_if_already_uploaded: + raise AlreadyExists(f'file {self.name} already exists in course {course.name}, but you don\'t want to overwrite.') + else: + file_on_canvas = find_file_in_course(self.givenpath,course) + else: + # get the remote file + print(f'file not already uploaded, uploading {self.name}') + + success_code, json_response = course.upload(self.givenpath, parent_folder_path=dest,on_duplicate=on_duplicate) + file_on_canvas = course.get_file(json_response['id']) + if not success_code: + print(f'failed to upload... {self.givenpath}') + + + self.canvas_obj = file_on_canvas + + return file_on_canvas + + + + + def make_href_url(self,courseid): + """ + constructs a string which can be used to reference the file in a Canvas page. + + sadly, the JSON back from Canvas doesn't just produce this for us. lame. + + """ + import canvasapi + file = self.canvas_obj + assert(isinstance(self.canvas_obj, canvasapi.file.File)) + + n = file.url.find('/files') + + url = file.url[:n]+'/courses/'+str(courseid)+'/files/'+str(file.id)+'/download?wrap=1' + + return url + + + def make_api_endpoint_url(self,courseid): + import canvasapi + file = self.canvas_obj + assert(isinstance(self.canvas_obj, canvasapi.file.File)) + + n = file.url.find('/files') + + url = file.url[:n] + '/api/v1/courses/' + str(courseid) + '/files/' + str(file.id) + return url + # data-api-endpoint="https://uws-td.instructure.com/api/v1/courses/3099/files/219835" + + + def __str__(self): + result = "\n" + result = result + f'givenpath: {self.givenpath}\n' + result = result + f'name: {self.name}\n' + result = result + f'folder: {self.folder}\n' + result = result + f'alttext: {self.alttext}\n' + result = result + f'canvas_obj: {self.canvas_obj}\n' + url = self.make_href_url('fakecoursenumber') + result = result + f'constructed canvas url: {url}\n' + + return result+'\n' + + def __repr__(self): + return str(self) + + + +class Link(CanvasObject): + """ + a containerization of url's, for uploading to Canvas modules + """ + def __init__(self, folder): + super(Link, self).__init__() + self.folder = folder + + import json, os + from os.path import join + + self.metaname = path.join(folder,'meta.json') + with open(self.metaname,'r',encoding='utf-8') as f: + self.metadata = json.load(f) + + if 'indent' in self.metadata: + self.indent = self.metadata['indent'] + else: + self.indent = 0 + + def __str__(self): + result = f"Link({self.metadata['external_url']})" + return result + + def __repr__(self): + return str(self) + + + def publish(self, course, overwrite=False): + + for m in self.metadata['modules']: + if link_on_canvas:= self.is_in_module(course, m): + if not overwrite: + n = self.metadata['external_url'] + raise AlreadyExists(f'trying to upload {self}, but is already on Canvas in module {m}') + else: + link_on_canvas.edit(module_item={'external_url':self.metadata['external_url'],'title':self.metadata['name'], 'new_tab':bool(self.metadata['new_tab'])}) + + else: + mod = create_or_get_module(m, course) + mod.create_module_item(module_item={'type':'ExternalUrl','external_url':self.metadata['external_url'],'title':self.metadata['name'], 'new_tab':bool(self.metadata['new_tab']), 'indent':self.indent}) + + + def is_already_uploaded(self, course): + for m in self.metadata['modules']: + if not self.is_in_module(course, m): + return False + + return True + + + + def is_in_module(self, course, module_name): + try: + module = get_module(module_name,course) + except DoesntExist as e: + return None + + + for item in module.get_module_items(): + + if item.type=='ExternalUrl' and item.external_url==self.metadata['external_url']: + return item + else: + continue + + return None + + + + +class File(CanvasObject): + """ + a containerization of arbitrary files, for uploading to Canvas + """ + def __init__(self, folder): + super(File, self).__init__(folder) + + import json, os + from os.path import join + + self.folder = folder + + self.metaname = path.join(folder,'meta.json') + with open(self.metaname,'r',encoding='utf-8') as f: + self.metadata = json.load(f) + + try: + self.title = self.metadata['title'] + except: + self.title = self.metadata['filename'] + + + if 'indent' in self.metadata: + self.indent = self.metadata['indent'] + else: + self.indent = 0 + + + def __str__(self): + result = f"File({self.metadata})" + return result + + def __repr__(self): + return str(self) + + + def _upload_(self, course): + pass + + + def publish(self, course, overwrite=False): + """ + publishes a file to Canvas in a particular folder + """ + + on_duplicate='overwrite' + if (file_on_canvas:= self.is_already_uploaded(course)) and not overwrite: + # on_duplicate='rename' + n = self.metadata['filename'] + # content_id = file_on_canvas.id + + raise AlreadyExists(f'The file {n} is already on Canvas and `not overwrite`.') + else: + root = get_root_folder(course) + + d = self.metadata['destination'] + d = d.split('/') + + curr_dir = root + for subd in d: + try: + curr_dir = get_subfolder_named(curr_dir, subd) + except DoesntExist as e: + curr_dir = curr_dir.create_folder(subd) + + filepath_to_upload = path.join(self.folder,self.metadata['filename']) + reply = curr_dir.upload(file=filepath_to_upload,on_duplicate=on_duplicate) + + if not reply[0]: + raise RuntimeError(f'something went wrong uploading {filepath_to_upload}') + + file_on_canvas = reply[1] + content_id = file_on_canvas['id'] + + + # now to make sure it's in the right modules + for module_name in self.metadata['modules']: + module = create_or_get_module(module_name, course) + + items = module.get_module_items() + is_in = False + for item in items: + if item.type=='File' and item.content_id==content_id: + is_in = True + break + + if not is_in: + module.create_module_item(module_item={'type':'File', 'content_id':content_id, 'title':self.title, 'indent':self.indent}) + # if the title doesn't match, update it + elif item.title != self.title: + item.edit(module_item={'type':'File', 'content_id':content_id, 'title':self.title},module=module) + + + def is_in_module(self, course, module_name): + file_on_canvas = self.is_already_uploaded(course) + + if not file_on_canvas: + return False + + module = get_module(module_name,course) + + for item in module.get_module_items(): + + if item.type=='File' and item.content_id==file_on_canvas.id: + return True + else: + continue + + return False + + + def is_already_uploaded(self,course, require_same_path=True): + files = course.get_files() + + for f in files: + if f.filename == self.metadata['filename']: + + if not require_same_path: + return f + else: + containing_folder = course.get_folder(f.folder_id) + if containing_folder.full_name.startswith('course files') and containing_folder.full_name.endswith(self.metadata['destination']): + return f + + + return None + + diff --git a/markdown2canvas/exception/__init__.py b/markdown2canvas/exception/__init__.py new file mode 100644 index 0000000..209c61b --- /dev/null +++ b/markdown2canvas/exception/__init__.py @@ -0,0 +1,29 @@ +class AlreadyExists(Exception): + + def __init__(self, message, errors=""): + # Call the base class constructor with the parameters it needs + super().__init__(message) + + self.errors = errors + +class SetupError(Exception): + + def __init__(self, message, errors=""): + # Call the base class constructor with the parameters it needs + super().__init__(message) + + self.errors = errors + + + + +class DoesntExist(Exception): + """ + Used when getting a thing, but it doesn't exist + """ + + def __init__(self, message, errors=""): + # Call the base class constructor with the parameters it needs + super().__init__(message) + + self.errors = errors \ No newline at end of file diff --git a/markdown2canvas/free_functions/__init__.py b/markdown2canvas/free_functions/__init__.py new file mode 100644 index 0000000..45ccd04 --- /dev/null +++ b/markdown2canvas/free_functions/__init__.py @@ -0,0 +1,597 @@ +import os.path as path +import os + +import logging +logger = logging.getLogger(__name__) + +import canvasapi + +def is_file_already_uploaded(filename,course): + """ + returns a boolean, true if there's a file of `filename` already in `course`. + + This function wants the full path to the file. + """ + return ( not find_file_in_course(filename,course) is None ) + + + + +def find_file_in_course(filename,course): + """ + Checks to see of the file at `filename` is already in the "files" part of `course`. + + It tests filename and size as reported on disk. If it finds a match, then it's up. + + This function wants the full path to the file. + """ + import os + + base = path.split(filename)[1] + + files = course.get_files() + for f in files: + if f.filename==base and f.size == path.getsize(filename): + return f + + return None + + + + + +def is_page_already_uploaded(name,course): + """ + returns a boolean indicating whether a page of the given `name` is already in the `course`. + """ + return ( not find_page_in_course(name,course) is None ) + + +def find_page_in_course(name,course): + """ + Checks to see if there's already a page named `name` as part of `course`. + + tests merely based on the name. assumes assignments are uniquely named. + """ + import os + pages = course.get_pages() + for p in pages: + if p.title == name: + return p + + return None + + + +def is_assignment_already_uploaded(name,course): + """ + returns a boolean indicating whether an assignment of the given `name` is already in the `course`. + """ + return ( not find_assignment_in_course(name,course) is None ) + + +def find_assignment_in_course(name,course): + """ + Checks to see if there's already an assignment named `name` as part of `course`. + + tests merely based on the name. assumes assingments are uniquely named. + """ + import os + assignments = course.get_assignments() + for a in assignments: + + if a.name == name: + return a + + return None + + + + +def get_canvas_key_url(): + """ + reads a file using an environment variable, namely the file specified in `CANVAS_CREDENTIAL_FILE`. + + We need the + + * API_KEY + * API_URL + + variables from that file. + """ + from os import environ + + cred_loc = environ.get('CANVAS_CREDENTIAL_FILE') + if cred_loc is None: + raise SetupError('`get_canvas_key_url()` needs an environment variable `CANVAS_CREDENTIAL_FILE`, containing the full path of the file containing your Canvas API_KEY, *including the file name*') + + # yes, this is scary. it was also low-hanging fruit, and doing it another way was going to be too much work + with open(path.join(cred_loc),encoding='utf-8') as cred_file: + exec(cred_file.read(),locals()) + + if isinstance(locals()['API_KEY'], str): + logger.info(f'using canvas with API_KEY as defined in {cred_loc}') + else: + raise SetupError(f'failing to use canvas. Make sure that file {cred_loc} contains a line of code defining a string variable `API_KEY="keyhere"`') + + return locals()['API_KEY'],locals()['API_URL'] + + +def make_canvas_api_obj(url=None): + """ + - reads the key from a python file, path to which must be in environment variable CANVAS_CREDENTIAL_FILE. + - optionally, pass in a url to use, in case you don't want the default one you put in your CANVAS_CREDENTIAL_FILE. + """ + + key, default_url = get_canvas_key_url() + + if not url: + url = default_url + + return canvasapi.Canvas(url, key) + + + +def generate_course_link(type,name,all_of_type,courseid=None): + ''' + Given a type (assignment or page) and the name of said object, generate a link + within course to that object. + ''' + if type in ['page','quiz']: + the_item = next( (p for p in all_of_type if p.title == name) , None) + elif type == 'assignment': + the_item = next( (a for a in all_of_type if a.name == name) , None) + elif type == 'file': + the_item = next( (a for a in all_of_type if a.display_name == name) , None) + if the_item is None: # Separate case to allow change of filenames on Canvas to names that did exist + the_item = next( (a for a in all_of_type if a.filename == name) , None) + # Canvas retains the name of the file uploaded and calls it `filename`. + # To access the name of the document seen in the Course Files, we use `display_name`. + else: + the_item = None + + + if the_item is None: + print(f"âšī¸ No content of type `{type}` named `{name}` exists in this Canvas course. Either you have the name incorrect, the content is not yet uploaded, or you used incorrect type before the colon") + elif type == 'file' and not courseid is None: + # Construct the url with reference to the coruse its coming from + file_id = the_item.id + full_url = the_item.url + stopper = full_url.find("files") + + html_url = full_url[:stopper] + "courses/" + str(courseid) + "/files/" + str(file_id) + + return html_url + elif type == 'file': + # Construct the url - removing the "download" portion + full_url = the_item.url + stopper = full_url.find("download") + return full_url[:stopper] + else: + return the_item.html_url + + + +def find_in_containing_directory_path(target): + import pathlib + + target = pathlib.Path(target) + + here = pathlib.Path('.').absolute() + + testme = here / target + + found = testme.exists() + + while (not found) and here.parent!=here: + here = here.parent + testme = here / target + found = testme.exists() + + + if not found: + raise FileNotFoundError('unable to find {} in a containing folder of {}'.format(target, pathlib.Path('.').absolute())) + + return here / target + + + +def preprocess_replacements(contents, replacements_path): + """ + attempts to read in a file containing substitutions to make, and then makes those substitutions + """ + + if replacements_path is None: + return contents + with open(replacements_path,'r',encoding='utf-8') as f: + import json + replacements = json.loads(f.read()) + + for source, target in replacements.items(): + contents = contents.replace(source, target) + + return contents + + + + +def preprocess_markdown_images(contents,style_path): + + rel_style_path = find_in_containing_directory_path(style_path) + + contents = contents.replace('$PATHTOMD2CANVASSTYLEFILE',str(rel_style_path)) + + return contents + + +def get_default_property(key, helpstr): + + defaults_name = find_in_containing_directory_path(path.join("_course_metadata","defaults.json")) + + try: + logger.info(f'trying to use defaults from {defaults_name}') + with open(defaults_name,'r',encoding='utf-8') as f: + import json + defaults = json.loads(f.read()) + + if key in defaults: + return defaults[key] + else: + print(f'no default `{key}` specified in {defaults_name}. add an entry with key `{key}`, being {helpstr}') + return None + + except Exception as e: + print(f'WARNING: failed to load defaults from `{defaults_name}`. either you are not at the correct location to be doing this, or you need to create a json file at {defaults_name}.') + return None + + +def get_default_style_name(): + return get_default_property(key='style', helpstr='a path to a file relative to the top course folder') + +def get_default_replacements_name(): + return get_default_property(key='replacements', helpstr='a path to a json file containing key:value pairs of text-to-replace. this path should be expressed relative to the top course folder') + + + + +def apply_style_markdown(sourcename, style_path, outname): + from os.path import join + + # need to add header and footer. assume they're called `header.md` and `footer.md`. we're just going to concatenate them and dump to file. + + with open(sourcename,'r',encoding='utf-8') as f: + body = f.read() + + with open(join(style_path,'header.md'),'r',encoding='utf-8') as f: + header = f.read() + + with open(join(style_path,'footer.md'),'r',encoding='utf-8') as f: + footer = f.read() + + + contents = f'{header}\n{body}\n{footer}' + contents = preprocess_markdown_images(contents, style_path) + + with open(outname,'w',encoding='utf-8') as f: + f.write(contents) + + + + +def apply_style_html(translated_html_without_hf, style_path, outname): + from os.path import join + + # need to add header and footer. assume they're called `header.html` and `footer.html`. we're just going to concatenate them and dump to file. + + with open(join(style_path,'header.html'),'r',encoding='utf-8') as f: + header = f.read() + + with open(join(style_path,'footer.html'),'r',encoding='utf-8') as f: + footer = f.read() + + + return f'{header}\n{translated_html_without_hf}\n{footer}' + + + + + +def markdown2html(filename, course, replacements_path): + """ + This is the main routine in the library. + + This function returns a string of html code. + + It does replacements, emojizes, converts markdown-->html via `markdown.markdown`, and does page, assignment, and file reference link adjustments. + + If `course` is None, then you won't get some of the functionality. In particular, you won't get link replacements for references to other content on Canvas. + + If `replacements_path` is None, then no replacements, duh. Otherwise it should be a string or Path object to an existing json file containing key-value pairs of strings to replace with other strings. + """ + if course is None: + courseid = None + else: + courseid = course.id + + root = path.split(filename)[0] + + import emoji + import markdown + from bs4 import BeautifulSoup + + + with open(filename,'r',encoding='utf-8') as file: + markdown_source = file.read() + + markdown_source = preprocess_replacements(markdown_source, replacements_path) + + emojified = emoji.emojize(markdown_source) + + + html = markdown.markdown(emojified, extensions=['codehilite','fenced_code','md_in_html','tables','nl2br']) # see https://python-markdown.github.io/extensions/ + soup = BeautifulSoup(html,features="lxml") + + all_imgs = soup.findAll("img") + for img in all_imgs: + src = img["src"] + if ('http://' not in src) and ('https://' not in src): + img["src"] = path.join(root,src) + + all_links = soup.findAll("a") + course_page_and_assignments = {} + if any(l['href'].startswith("page:") for l in all_links) and course: + course_page_and_assignments['page'] = course.get_pages() + if any(l['href'].startswith("assignment:") for l in all_links) and course: + course_page_and_assignments['assignment'] = course.get_assignments() + if any(l['href'].startswith("quiz:") for l in all_links) and course: + course_page_and_assignments['quiz'] = course.get_quizzes() + if any(l['href'].startswith("file:") for l in all_links) and course: + course_page_and_assignments['file'] = course.get_files() + for f in all_links: + href = f["href"] + root_href = path.join(root,href) + split_at_colon = href.split(":",1) + if path.exists(path.abspath(root_href)): + f["href"] = root_href + elif course and split_at_colon[0] in ['assignment','page','quiz','file']: + type = split_at_colon[0] + name = split_at_colon[1].strip() + get_link = generate_course_link(type,name,course_page_and_assignments[type],courseid) + if get_link: + f["href"] = get_link + + + return str(soup) + + + + + +def find_local_images(html): + """ + constructs a map of local url's : Images + """ + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html,features="lxml") + + local_images = {} + + all_imgs = soup.findAll("img") + + if all_imgs: + for img in all_imgs: + src = img["src"] + if src[:7] not in ['https:/','http://']: + local_images[src] = Image(path.abspath(src)) + + return local_images + + + + + +def adjust_html_for_images(html, published_images, courseid): + """ + + published_images: a dict of Image objects, which should have been published (so we have their canvas objects stored into them) + + this function edits the html source, replacing local url's + with url's to images on Canvas. + """ + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html,features="lxml") + + all_imgs = soup.findAll("img") + if all_imgs: + for img in all_imgs: + src = img["src"] + if src[:7] not in ['https:/','http://']: + # find the image in the list of published images, replace url, do more stuff. + local_img = published_images[src] + img['src'] = local_img.make_src_url(courseid) + img['class'] = "instructure_file_link inline_disabled" + img['data-api-endpoint'] = local_img.make_api_endpoint_url(courseid) + img['data-api-returntype'] = 'File' + + return str(soup) + + #+ # + #
+ + + + +def find_local_files(html): + """ + constructs a list of BareFiles, so that they can later be replaced with a url to a canvas thing + """ + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html,features="lxml") + + local_files = {} + + all_links = soup.findAll("a") + + if all_links: + for file in all_links: + href = file["href"] + if path.exists(path.abspath(href)): + local_files[href] = BareFile(path.abspath(href)) + + return local_files + + + +def adjust_html_for_files(html, published_files, courseid): + + + # need to write a url like this : + # Download + + + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html,features="lxml") + + all_files = soup.findAll("a") + + if all_files: + for file in all_files: + href = file["href"] + if path.exists(path.abspath(href)): + # find the image in the list of published images, replace url, do more stuff. + local_file = published_files[href] + file['href'] = local_file.make_href_url(courseid) + file['class'] = "instructure_file_link instructure_scribd_file" + file['title'] = local_file.name # what it's called when you download it??? + file['data-api-endpoint'] = local_file.make_api_endpoint_url(courseid) + file['data-api-returntype'] = 'File' + + return str(soup) + + + +def get_root_folder(course): + for f in course.get_folders(): + if f.full_name == 'course files': + return f + + + + + + + + + +def get_assignment_group_id(assignment_group_name, course, create_if_necessary=False): + + existing_groups = course.get_assignment_groups() + + if not isinstance(assignment_group_name,str): + raise RuntimeError(f'assignment_group_name must be a string, but I got {assignment_group_name} of type {type(assignment_group_name)}') + + + for g in existing_groups: + if g.name == assignment_group_name: + return g.id + + + + if create_if_necessary: + msg = f'making new assignment group `{assignment_group_name}`' + logger.info(msg) + + group = course.create_assignment_group(name=assignment_group_name) + group.edit(name=assignment_group_name) # this feels stupid. didn't i just request its name be this? + + return group.id + else: + raise DoesntExist(f'cannot get assignment group id because an assignment group of name {assignment_group_name} does not already exist, and `create_if_necessary` is set to False') + + + + + +def create_or_get_assignment(name, course, even_if_exists = False): + + if is_assignment_already_uploaded(name,course): + if even_if_exists: + return find_assignment_in_course(name,course) + else: + raise AlreadyExists(f"assignment {name} already exists") + else: + # make new assignment of name in course. + return course.create_assignment(assignment={'name':name}) + + + +def create_or_get_page(name, course, even_if_exists): + if is_page_already_uploaded(name,course): + + if even_if_exists: + return find_page_in_course(name,course) + else: + raise AlreadyExists(f"page {name} already exists") + else: + # make new assignment of name in course. + result = course.create_page(wiki_page={'body':"empty page",'title':name}) + return result + + + + +def create_or_get_module(module_name, course): + + try: + return get_module(module_name, course) + except DoesntExist as e: + return course.create_module(module={'name':module_name}) + + + + +def get_module(module_name, course): + """ + returns + * Module if such a module exists, + * raises if not + """ + modules = course.get_modules() + + for m in modules: + if m.name == module_name: + return m + + raise DoesntExist(f"tried to get module {module_name}, but it doesn't exist in the course") + + +def get_subfolder_named(folder, subfolder_name): + + assert '/' not in subfolder_name, "this is likely broken if subfolder has a / in its name, / gets converted to something else by Canvas. don't use / in subfolder names, that's not allowed" + + current_subfolders = folder.get_folders() + for f in current_subfolders: + if f.name == subfolder_name: + return f + + raise DoesntExist(f'a subfolder of {folder.name} named {subfolder_name} does not currently exist') + + +def delete_module(module_name, course, even_if_exists): + + if even_if_exists: + try: + m = get_module(module_name, course) + m.delete() + except DoesntExist as e: + return + + else: + # this path is expected to raise if the module doesn't exist + m = get_module(module_name, course) + m.delete() From 9b85ee978e3188f2356fac23869a9c82bf69b140 Mon Sep 17 00:00:00 2001 From: silviana amethyst <1388063+ofloveandhate@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:43:33 -0500 Subject: [PATCH 14/57] refactor completed? namespace is so much cleaner. added module level docstrings docstrings for most functions added all unit tests passed --- docs/{classes.rst => canvas_objects.rst} | 16 +- docs/exceptions.rst | 6 +- docs/free_functions.rst | 29 +- docs/markdown2canvas.rst | 2 +- docs/requirements.txt | 3 +- markdown2canvas/__init__.py | 65 ++-- markdown2canvas/base_classes/__init__.py | 235 ------------- .../__init__.py => canvas2markdown.py} | 108 ++++-- .../__init__.py => canvas_objects.py} | 327 +++++++++++++++++- .../course_interaction_functions.py | 280 +++++++++++++++ markdown2canvas/exception.py | 45 +++ markdown2canvas/exception/__init__.py | 29 -- markdown2canvas/logging.py | 46 +++ markdown2canvas/setup_functions.py | 68 ++++ markdown2canvas/{tool/__init__.py => tool.py} | 10 +- .../__init__.py => translation_functions.py} | 309 ++--------------- test/course_id.py | 2 +- test/test_assignment.py | 2 +- test/test_download_pages_to_markdown.py | 4 +- test/test_droplets.py | 2 +- test/test_file.py | 2 +- test/test_link.py | 2 +- test/test_link_to_local_file.py | 2 +- test/test_page.py | 2 +- test/test_page_in_module.py | 2 +- test/test_style.py | 2 +- 26 files changed, 958 insertions(+), 642 deletions(-) rename docs/{classes.rst => canvas_objects.rst} (54%) delete mode 100644 markdown2canvas/base_classes/__init__.py rename markdown2canvas/{canvas2markdown/__init__.py => canvas2markdown.py} (62%) rename markdown2canvas/{classes/__init__.py => canvas_objects.py} (69%) create mode 100644 markdown2canvas/course_interaction_functions.py create mode 100644 markdown2canvas/exception.py delete mode 100644 markdown2canvas/exception/__init__.py create mode 100644 markdown2canvas/logging.py create mode 100644 markdown2canvas/setup_functions.py rename markdown2canvas/{tool/__init__.py => tool.py} (89%) rename markdown2canvas/{free_functions/__init__.py => translation_functions.py} (57%) diff --git a/docs/classes.rst b/docs/canvas_objects.rst similarity index 54% rename from docs/classes.rst rename to docs/canvas_objects.rst index 7d056da..6529ff4 100644 --- a/docs/classes.rst +++ b/docs/canvas_objects.rst @@ -1,5 +1,5 @@ -Concrete Classes -------------------- +Concrete Classes for Canvas Objects +-------------------------------------- @@ -12,6 +12,14 @@ Concrete Classes :members: :undoc-members: +.. autoclass:: markdown2canvas.Image + :members: + :undoc-members: + + +.. autoclass:: markdown2canvas.Link + :members: + :undoc-members: .. autoclass:: markdown2canvas.File @@ -32,11 +40,11 @@ Concrete Classes Base Classes -------------- -.. autoclass:: markdown2canvas.CanvasObject +.. autoclass:: markdown2canvas.canvas_objects.CanvasObject :members: :undoc-members: -.. autoclass:: markdown2canvas.Document +.. autoclass:: markdown2canvas.canvas_objects.Document :members: :undoc-members: \ No newline at end of file diff --git a/docs/exceptions.rst b/docs/exceptions.rst index 9279079..1cb36b9 100644 --- a/docs/exceptions.rst +++ b/docs/exceptions.rst @@ -1,14 +1,14 @@ Exceptions ------------- -.. autoclass:: markdown2canvas.AlreadyExists +.. autoclass:: markdown2canvas.exception.AlreadyExists :members: :undoc-members: -.. autoclass:: markdown2canvas.SetupError +.. autoclass:: markdown2canvas.exception.SetupError :members: :undoc-members: -.. autoclass:: markdown2canvas.DoesntExist +.. autoclass:: markdown2canvas.exception.DoesntExist :members: :undoc-members: \ No newline at end of file diff --git a/docs/free_functions.rst b/docs/free_functions.rst index e2a292e..0c0c3de 100644 --- a/docs/free_functions.rst +++ b/docs/free_functions.rst @@ -1,3 +1,30 @@ Free functions ------------------- +===================== + + +Setup functions +------------------- + +.. automodule:: markdown2canvas.setup_functions + :members: + + + +Functions for interacting with a course on Canvas +--------------------------------------------------- + +.. automodule:: markdown2canvas.course_interaction_functions + :members: + + +Functions markdown2canvas uses to translate from markdown to Canvas-html +-------------------------------------------------------------------------- + +.. automodule:: markdown2canvas.translation_functions + :members: + +.. autofunction:: markdown2canvas.canvas_objects.find_local_images + +.. autofunction:: markdown2canvas.canvas_objects.find_local_files + diff --git a/docs/markdown2canvas.rst b/docs/markdown2canvas.rst index 6b89fd0..49b26ba 100644 --- a/docs/markdown2canvas.rst +++ b/docs/markdown2canvas.rst @@ -12,7 +12,7 @@ Python libraries used: `canvasapi` .. toctree:: :maxdepth: 2 - classes + canvas_objects exceptions free_functions diff --git a/docs/requirements.txt b/docs/requirements.txt index ddc2a8f..c1f96fb 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ canvasapi -Pygments \ No newline at end of file +Pygments +markdown \ No newline at end of file diff --git a/markdown2canvas/__init__.py b/markdown2canvas/__init__.py index 6280eb7..9e53524 100644 --- a/markdown2canvas/__init__.py +++ b/markdown2canvas/__init__.py @@ -1,60 +1,59 @@ -import canvasapi -import os.path as path -import os -import requests +""" +`markdown2canvas`, a library for containerizing and publishing Canvas content. +Containerization of content is via filesystem folders with a `meta.json` file specifying type of content. Some content types like Assignment and Page use `source.md`, while others like File and Image are just a `meta.json` plus the files. +Publishing content is via the `.publish` member function for the canvas object, like -import logging +``` +my_assignment.publish(course) +``` -# logging.basicConfig(encoding='utf-8') +Documentation may be found at the GitHub pages for this library. Use it. -import datetime -today = datetime.datetime.today().strftime("%Y-%m-%d") -log_level=logging.DEBUG +A more complete example might be -log_dir = path.join(path.normpath(os.getcwd()), '_logs') +``` +import markdown2canvas as mc +canvas_url = "https://uweau.instructure.com/" # đ¯ REPLACE WITH YOUR URL -if not path.exists(log_dir): - os.mkdir(log_dir) +# get the course. +course_id = 705022 # đ¯ REPLACE WITH YOUR NUMBER!!!!!!!!!!!!!!!!! +course = canvas.get_course(course_id) -log_filename = path.join(log_dir, f'markdown2canvas_{today}.log') +# make the API object. this is from the `canvasapi` library, NOT something in `markdown2canvas`. +canvas = mc.make_canvas_api_obj(url=canvas_url) +my_assignment = mc.Assignent('path_to_assignment") -log_encoding = 'utf-8' +# finally, publish +my_assignment.publish(course) +``` +""" -root_logger = logging.getLogger() -root_logger.setLevel(log_level) -handler = logging.FileHandler(log_filename, 'a', log_encoding) -root_logger.addHandler(handler) +# the root-level file for `markdown2canvas` -logging.debug(f'starting logging at {datetime.datetime.now()}') +__version__ = '0.' +__author__ = 'silviana amethyst, Mckenzie West, Allison Beemer' -logging.debug(f'reducing logging level of `requests` to WARNING') -logging.getLogger('canvasapi.requester').setLevel(logging.WARNING) -logging.getLogger('requests').setLevel(logging.WARNING) +import markdown2canvas.logging +import markdown2canvas.exception +from markdown2canvas.setup_functions import * +import markdown2canvas.translation_functions - -from markdown2canvas.exception import AlreadyExists, SetupError, DoesntExist - - -from markdown2canvas.free_functions import * - +from markdown2canvas.course_interaction_functions import * ################## classes -from markdown2canvas.base_classes import CanvasObject, Document -from markdown2canvas.classes import Page, Assignment, Image, BareFile, Link, File - - - +from markdown2canvas.canvas_objects import CanvasObject, Document, Page, Assignment, Image, File, BareFile, Link +import markdown2canvas.canvas2markdown import markdown2canvas.tool diff --git a/markdown2canvas/base_classes/__init__.py b/markdown2canvas/base_classes/__init__.py deleted file mode 100644 index cc04458..0000000 --- a/markdown2canvas/base_classes/__init__.py +++ /dev/null @@ -1,235 +0,0 @@ - -import os.path as path -import os - -import canvasapi -from markdown2canvas.free_functions import * - -class CanvasObject(object): - """ - A base class for wrapping canvas objects. - """ - - def __init__(self,canvas_obj=None): - - super(object, self).__init__() - - self.canvas_obj = canvas_obj - - - - -class Document(CanvasObject): - """ - A base class which handles common pieces of interface for things like Pages and Assignments - - This type is abstract. Assignments and Pages both derive from this. - - At least two files are required in the folder for a Document: - - 1. `meta.json` - 2. `source.md` - - You may have additional files in the folder for a Document, such as images and files to include in the content on Canvas. This library will automatically upload those for you! - """ - - def __init__(self,folder,course=None): - """ - Construct a Document. - Reads the meta.json file and source.md files - from the specified folder. - """ - - super(Document,self).__init__(folder) - import json, os - from os.path import join - - self.folder = folder - - # try to open, and see if the meta and source files exist. - # if not, raise - self.metaname = path.join(folder,'meta.json') - with open(self.metaname,'r',encoding='utf-8') as f: - self.metadata = json.load(f) - - self.sourcename = path.join(folder,'source.md') - - - # variables populated from the metadata. should these even exist? IDK - self.name = None - self.style_path = None - self.replacements_path = None - - # populate the above variables from the meta.json file - self._set_from_metadata() - - - # these internally-used variables are used to carry state between functions - self._local_images = None - self._local_files = None - self._translated_html = None - - - def _set_from_metadata(self): - """ - this function is called during `__init__`. - """ - - self.name = self.metadata['name'] - - if 'modules' in self.metadata: - self.modules = self.metadata['modules'] - else: - self.modules = [] - - if 'indent' in self.metadata: - self.indent = self.metadata['indent'] - else: - self.indent = 0 - - if 'style' in self.metadata: - self.style_path = find_in_containing_directory_path(self.metadata['style']) - else: - self.style_path = get_default_style_name() # could be None if doesn't exist - if self.style_path: - self.style_path = find_in_containing_directory_path(self.style_path) - - if 'replacements' in self.metadata: - self.replacements_path = find_in_containing_directory_path(self.metadata['replacements']) - else: - self.replacements_path = get_default_replacements_name() # could be None if doesn't exist - if self.replacements_path: - self.replacements_path = find_in_containing_directory_path(self.replacements_path) - - - - - def translate_to_html(self,course): - """ - populates the internal variables with the results of translating from markdown to html. - - This step requires the `course` since this library allows for referencing of content already on canvas (or to be later published on Canvas) - - The main result of translation is held in self._translated_html. The local content (on YOUR computer, NOT Canvas) is parsed out and held in `self._local_images` and `self._local_files`. - - * This function does NOT make content appear on Canvas. - * It DOES leave behind a temporary file: `{folder}/styled_source.md`. Be sure to add `*/styled_source.md` to your .gitignore for your course! - """ - from os.path import join - - if self.style_path: - outname = join(self.folder,"styled_source.md") - apply_style_markdown(self.sourcename, self.style_path, outname) - - translated_html_without_hf = markdown2html(outname,course, self.replacements_path) - - self._translated_html = apply_style_html(translated_html_without_hf, self.style_path, outname) - else: - self._translated_html = markdown2html(self.sourcename,course, self.replacements_path) - - - self._local_images = find_local_images(self._translated_html) - self._local_files = find_local_files(self._translated_html) - - - - - - def publish_linked_content_and_adjust_html(self,course,overwrite=False): - """ - this function should be called *after* `translate_to_html`, since it requires the internal variables that other function populates - - the result of this function is written to `folder/result.html` - - * This function does NOT make content appear on Canvas. - * It DOES leave behind a temporary file: `{folder}/result.html`. Be sure to add `*/result.html` to your .gitignore for your course! - """ - - # first, publish the local images. - for im in self._local_images.values(): - im.publish(course,'images', overwrite=overwrite) - - for file in self._local_files.values(): - file.publish(course,'automatically_uploaded_files', overwrite=overwrite) - - - # then, deal with the urls - self._translated_html = adjust_html_for_images(self._translated_html, self._local_images, course.id) - self._translated_html = adjust_html_for_files(self._translated_html, self._local_files, course.id) - - save_location = path.join(self.folder,'result.html') - with open(save_location,'w',encoding='utf-8') as result: - result.write(self._translated_html) - - - - - - - - - - def _construct_dict_of_props(self): - """ - construct a dictionary of properties, such that it can be used to `edit` a canvas object. - """ - d = {} - return d - - - def ensure_in_modules(self, course): - """ - makes sure this item is listed in the Module on Canvas. If it's not, it's added to the bottom. There's not currently any way to control order. - - If the item doesn't already exist, this function will raise. Be sure to actually publish the content first. - """ - - if not self.canvas_obj: - raise DoesntExist(f"trying to make sure an object is in its modules, but this item ({self.name}) doesn't exist on canvas yet. publish it first.") - - for module_name in self.modules: - module = create_or_get_module(module_name, course) - - if not self.is_in_module(module_name, course): - - if self.metadata['type'] == 'page': - content_id = self.canvas_obj.page_id - elif self.metadata['type'] == 'assignment': - content_id = self.canvas_obj.id - - - module.create_module_item(module_item={'type':self.metadata['type'], 'content_id':content_id, 'indent':self.indent}) - - - def is_in_module(self, module_name, course): - """ - checks whether this content is an item in the listed module, where `module_name` is a string. It's case sensitive and exact. - - passthrough raise if the module doesn't exist - """ - - module = get_module(module_name,course) - - for item in module.get_module_items(): - - if item.type=='Page': - if self.metadata['type']=='page': - - if course.get_page(item.page_url).title == self.name: - return True - - else: - continue - - - if item.type=='Assignment': - if self.metadata['type']=='assignment': - - if course.get_assignment(assignment=item.content_id).name == self.name: - return True - else: - continue - - return False - - diff --git a/markdown2canvas/canvas2markdown/__init__.py b/markdown2canvas/canvas2markdown.py similarity index 62% rename from markdown2canvas/canvas2markdown/__init__.py rename to markdown2canvas/canvas2markdown.py index d3e6131..d1c53fb 100644 --- a/markdown2canvas/canvas2markdown/__init__.py +++ b/markdown2canvas/canvas2markdown.py @@ -1,11 +1,84 @@ +''' +Functions for grabbing non-containerized content from Canvas and saving it to disk. Useful for making a markdown2canvas repo from an existing course. + +The two main functions you should use are `download_pages` and `download_assignments`. + +The resulting folder will have name equal to the name of the content on Canvas, for better or for worse. + +I can see some situations where folder names are invalid -- feel free to improve this functionality. PR's welcome. + +''' + +__all__ = [ + 'download_pages','download_assignments' + 'page2markdown','assignment2markdown'] + + + import canvasapi +import os.path as path + +import logging +logger = logging.getLogger() + + + + +def download_pages(destination, course, even_if_exists=False, name_filter=None): + """ + downloads the regular pages from a course, saving them + into a markdown2canvas compatible format. that is, as + a folder with markdown source and json metadata. + + You can provide a predicate `name_filter` to filter on the name of the content! The function should return True/False. + + The flag `even_if_exists` is to overwrite the local content. If `even_if_exists` is True, the remote content will be written to disk EVEN IF IT ALREADY EXISTED LOCALLY. Thus, this may involve data loss. Use version control. + """ + + if name_filter is None: + name_filter = lambda x: True + + logger.info(f'downloading all pages from course {course.name}, saving to folder {destination}') + pages = course.get_pages() + for p in pages: + if name_filter(p.show_latest_revision().title): + page2markdown(destination,p,even_if_exists) + + +def download_assignments(destination, course, even_if_exists=False, name_filter=None): + """ + downloads the assignments from a course, saving them + into a markdown2canvas compatible format. that is, as + a folder with markdown source and json metadata. + + `destination` is the path you want to write the content to. This function will make sub-folders of `destination`. + + You can provide a predicate `name_filter` to filter on the name of the content! The function should return True/False. + + The flag `even_if_exists` is to overwrite the local content. If `even_if_exists` is True, the remote content will be written to disk EVEN IF IT ALREADY EXISTED LOCALLY. Thus, this may involve data loss. Use version control. + """ + + if name_filter is None: + name_filter = lambda x: True + + logger.info(f'downloading all pages from course {course.name}, saving to folder {destination}') + assignments = course.get_assignments() + for a in assignments: + if name_filter(a.name): + assignment2markdown(destination,a,even_if_exists) + + + def page2markdown(destination, page, even_if_exists=False): """ takes a Page from Canvas, and saves it to a folder inside `destination` into a markdown2canvas compatible format. - the folder is automatically named, at your own peril. + the folder is automatically named, at your own peril. Colons are removed, and spaces are replaced by underscores. + + The flag `even_if_exists` is to overwrite the local content. If `even_if_exists` is True, the remote content will be written to disk EVEN IF IT ALREADY EXISTED LOCALLY. Thus, this may involve data loss. Use version control. + """ import os @@ -47,21 +120,6 @@ def page2markdown(destination, page, even_if_exists=False): -def download_pages(destination, course, even_if_exists=False, name_filter=None): - """ - downloads the regular pages from a course, saving them - into a markdown2canvas compatible format. that is, as - a folder with markdown source and json metadata. - """ - - if name_filter is None: - name_filter = lambda x: True - - logger.info(f'downloading all pages from course {course.name}, saving to folder {destination}') - pages = course.get_pages() - for p in pages: - if name_filter(p.show_latest_revision().title): - page2markdown(destination,p,even_if_exists) def assignment2markdown(destination, assignment, even_if_exists=False): @@ -69,7 +127,9 @@ def assignment2markdown(destination, assignment, even_if_exists=False): takes a Page from Canvas, and saves it to a folder inside `destination` into a markdown2canvas compatible format. - the folder is automatically named, at your own peril. + the folder is automatically named, at your own peril. Colons are removed, and spaces are replaced by underscores. + + The flag `even_if_exists` is to overwrite the local content. If `even_if_exists` is True, the remote content will be written to disk EVEN IF IT ALREADY EXISTED LOCALLY. Thus, this may involve data loss. Use version control. """ import os @@ -106,18 +166,6 @@ def assignment2markdown(destination, assignment, even_if_exists=False): import json json.dump(d, file) -def download_assignments(destination, course, even_if_exists=False, name_filter=None): - """ - downloads the regular pages from a course, saving them - into a markdown2canvas compatible format. that is, as - a folder with markdown source and json metadata. - """ - if name_filter is None: - name_filter = lambda x: True - logger.info(f'downloading all pages from course {course.name}, saving to folder {destination}') - assignments = course.get_assignments() - for a in assignments: - if name_filter(a.name): - assignment2markdown(destination,a,even_if_exists) + diff --git a/markdown2canvas/classes/__init__.py b/markdown2canvas/canvas_objects.py similarity index 69% rename from markdown2canvas/classes/__init__.py rename to markdown2canvas/canvas_objects.py index 99f8371..1b4494c 100644 --- a/markdown2canvas/classes/__init__.py +++ b/markdown2canvas/canvas_objects.py @@ -1,10 +1,329 @@ +""" +Main content types for markdown2canvas + +Page and Assignment both require meta.json and source.md + +Image, File, BareFile each require meta.json and the file they containerize. Importantly, the file does NOT need to be contained in the folder!!! I'm trying to make your life easy. + +Link requires meta.json and that's it +""" + + +__all__ = [ + 'Page', + 'Assignment', + 'Image', + 'BareFile', + 'Link', + 'File' +] -import canvasapi import os.path as path import os -from markdown2canvas.base_classes import Document, CanvasObject -from markdown2canvas.free_functions import * +import canvasapi +from markdown2canvas.logging import * +from markdown2canvas.setup_functions import * +from markdown2canvas.translation_functions import * +from markdown2canvas.course_interaction_functions import * +from markdown2canvas.exception import AlreadyExists, SetupError, DoesntExist + + + + + +def find_local_images(html): + """ + constructs a map of local url's : Images + """ + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html,features="lxml") + + local_images = {} + + all_imgs = soup.findAll("img") + + if all_imgs: + for img in all_imgs: + src = img["src"] + if src[:7] not in ['https:/','http://']: + local_images[src] = Image(path.abspath(src)) + + return local_images + + + +def find_local_files(html): + """ + constructs a list of BareFiles, so that they can later be replaced with a url to a canvas thing + """ + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html,features="lxml") + + local_files = {} + + all_links = soup.findAll("a") + + if all_links: + for file in all_links: + href = file["href"] + if path.exists(path.abspath(href)): + local_files[href] = BareFile(path.abspath(href)) + + return local_files + + +class CanvasObject(object): + """ + A base class for wrapping canvas objects. + """ + + def __init__(self,canvas_obj=None): + + super(object, self).__init__() + + self.canvas_obj = canvas_obj + + + + +class Document(CanvasObject): + """ + A base class which handles common pieces of interface for things like Pages and Assignments + + This type is abstract. Assignments and Pages both derive from this. + + At least two files are required in the folder for a Document: + + 1. `meta.json` + 2. `source.md` + + You may have additional files in the folder for a Document, such as images and files to include in the content on Canvas. This library will automatically upload those for you! + """ + + def __init__(self,folder,course=None): + """ + Construct a Document. + Reads the meta.json file and source.md files + from the specified folder. + """ + + super(Document,self).__init__(folder) + import json, os + from os.path import join + + self.folder = folder + + # try to open, and see if the meta and source files exist. + # if not, raise + self.metaname = path.join(folder,'meta.json') + with open(self.metaname,'r',encoding='utf-8') as f: + self.metadata = json.load(f) + + self.sourcename = path.join(folder,'source.md') + + + # variables populated from the metadata. should these even exist? IDK + self.name = None + self.style_path = None + self.replacements_path = None + + # populate the above variables from the meta.json file + self._set_from_metadata() + + + # these internally-used variables are used to carry state between functions + self._local_images = None + self._local_files = None + self._translated_html = None + + + def _set_from_metadata(self): + """ + this function is called during `__init__`. + """ + + self.name = self.metadata['name'] + + if 'modules' in self.metadata: + self.modules = self.metadata['modules'] + else: + self.modules = [] + + if 'indent' in self.metadata: + self.indent = self.metadata['indent'] + else: + self.indent = 0 + + if 'style' in self.metadata: + self.style_path = find_in_containing_directory_path(self.metadata['style']) + else: + self.style_path = get_default_style_name() # could be None if doesn't exist + if self.style_path: + self.style_path = find_in_containing_directory_path(self.style_path) + + if 'replacements' in self.metadata: + self.replacements_path = find_in_containing_directory_path(self.metadata['replacements']) + else: + self.replacements_path = get_default_replacements_name() # could be None if doesn't exist + if self.replacements_path: + self.replacements_path = find_in_containing_directory_path(self.replacements_path) + + + + + def translate_to_html(self,course): + """ + populates the internal variables with the results of translating from markdown to html. + + This step requires the `course` since this library allows for referencing of content already on canvas (or to be later published on Canvas) + + The main result of translation is held in self._translated_html. The local content (on YOUR computer, NOT Canvas) is parsed out and held in `self._local_images` and `self._local_files`. + + * This function does NOT make content appear on Canvas. + * It DOES leave behind a temporary file: `{folder}/styled_source.md`. Be sure to add `*/styled_source.md` to your .gitignore for your course! + """ + from os.path import join + + if self.style_path: + outname = join(self.folder,"styled_source.md") + apply_style_markdown(self.sourcename, self.style_path, outname) + + translated_html_without_hf = markdown2html(outname,course, self.replacements_path) + + self._translated_html = apply_style_html(translated_html_without_hf, self.style_path, outname) + else: + self._translated_html = markdown2html(self.sourcename,course, self.replacements_path) + + + self._local_images = find_local_images(self._translated_html) + self._local_files = find_local_files(self._translated_html) + + + + + + def publish_linked_content_and_adjust_html(self,course,overwrite=False): + """ + this function should be called *after* `translate_to_html`, since it requires the internal variables that other function populates + + the result of this function is written to `folder/result.html` + + * This function does NOT make content appear on Canvas. + * It DOES leave behind a temporary file: `{folder}/result.html`. Be sure to add `*/result.html` to your .gitignore for your course! + """ + + # first, publish the local images. + for im in self._local_images.values(): + im.publish(course,'images', overwrite=overwrite) + + for file in self._local_files.values(): + file.publish(course,'automatically_uploaded_files', overwrite=overwrite) + + + # then, deal with the urls + self._translated_html = adjust_html_for_images(self._translated_html, self._local_images, course.id) + self._translated_html = adjust_html_for_files(self._translated_html, self._local_files, course.id) + + save_location = path.join(self.folder,'result.html') + with open(save_location,'w',encoding='utf-8') as result: + result.write(self._translated_html) + + + + + + + + + + def _construct_dict_of_props(self): + """ + construct a dictionary of properties, such that it can be used to `edit` a canvas object. + """ + d = {} + return d + + + def ensure_in_modules(self, course): + """ + makes sure this item is listed in the Module on Canvas. If it's not, it's added to the bottom. There's not currently any way to control order. + + If the item doesn't already exist, this function will raise. Be sure to actually publish the content first. + """ + + if not self.canvas_obj: + raise DoesntExist(f"trying to make sure an object is in its modules, but this item ({self.name}) doesn't exist on canvas yet. publish it first.") + + for module_name in self.modules: + module = create_or_get_module(module_name, course) + + if not self.is_in_module(module_name, course): + + if self.metadata['type'] == 'page': + content_id = self.canvas_obj.page_id + elif self.metadata['type'] == 'assignment': + content_id = self.canvas_obj.id + + + module.create_module_item(module_item={'type':self.metadata['type'], 'content_id':content_id, 'indent':self.indent}) + + + def is_in_module(self, module_name, course): + """ + checks whether this content is an item in the listed module, where `module_name` is a string. It's case sensitive and exact. + + passthrough raise if the module doesn't exist + """ + + module = get_module(module_name,course) + + for item in module.get_module_items(): + + if item.type=='Page': + if self.metadata['type']=='page': + + if course.get_page(item.page_url).title == self.name: + return True + + else: + continue + + + if item.type=='Assignment': + if self.metadata['type']=='assignment': + + if course.get_assignment(assignment=item.content_id).name == self.name: + return True + else: + continue + + return False + + + + + + + + + + + + + + + + + + + + + + + class Page(Document): """ @@ -649,5 +968,3 @@ def is_already_uploaded(self,course, require_same_path=True): return None - - diff --git a/markdown2canvas/course_interaction_functions.py b/markdown2canvas/course_interaction_functions.py new file mode 100644 index 0000000..1d9d771 --- /dev/null +++ b/markdown2canvas/course_interaction_functions.py @@ -0,0 +1,280 @@ +""" +Functions for making or getting things in Canvas, mostly by names-as-strings. + +Note that `canvasapi` mostly uses numeric identifiers to get things. This annoyed me and so I wrote these functions. + +These functions do NOT require containerized content, so these functions are probably useful even without a containerized course using markdown2canvas. +""" + +__all__ = [ + 'is_file_already_uploaded', + 'find_file_in_course', + 'is_page_already_uploaded', + 'find_page_in_course', + 'is_assignment_already_uploaded', + 'find_assignment_in_course', + 'get_root_folder', + 'get_assignment_group_id', + 'create_or_get_assignment', + 'create_or_get_page', + 'create_or_get_module', + 'get_module', + 'get_subfolder_named', + 'delete_module' + ] + + +from markdown2canvas.exception import * +import canvasapi + +import os.path as path + + + + + + +def is_file_already_uploaded(filename,course): + """ + returns a boolean, true if there's a file of `filename` already in `course`. + + This function wants the full path to the file. + + See also `find_file_in_course` + """ + return ( not find_file_in_course(filename,course) is None ) + + + + +def find_file_in_course(filename,course): + """ + Checks to see of the file at `filename` is already in the "files" part of `course`. + + It tests filename and size as reported on disk. If it finds a match, then it's up. + + This function wants the full path to the file. + + Note that `canvasapi` does NOT differentiate + between files in different "folders" on Canvas, + so if you have multiple files of the same name, + this will find the first one that matches both name and size. + """ + import os + + base = path.split(filename)[1] + + files = course.get_files() + for f in files: + if f.filename==base and f.size == path.getsize(filename): + return f + + return None + + + + + +def is_page_already_uploaded(name,course): + """ + returns a boolean indicating whether a page of the given `name` is already in the `course`. + """ + return ( not find_page_in_course(name,course) is None ) + + +def find_page_in_course(name,course): + """ + Checks to see if there's already a page named `name` as part of `course`. + + tests merely based on the name. assumes assignments are uniquely named. + """ + + import os + pages = course.get_pages() + for p in pages: + if p.title == name: + return p + + return None + + + +def is_assignment_already_uploaded(name,course): + """ + returns a boolean indicating whether an assignment of the given `name` is already in the `course`. + """ + return ( not find_assignment_in_course(name,course) is None ) + + +def find_assignment_in_course(name,course): + """ + Checks to see if there's already an assignment named `name` as part of `course`. + + Tests merely based on the name. assumes assingments are uniquely named. + """ + import os + assignments = course.get_assignments() + for a in assignments: + + if a.name == name: + return a + + return None + + + + + +def get_root_folder(course): + """ + gets the Folder object at root level in your course. + """ + + for f in course.get_folders(): + if f.full_name == 'course files': + return f + + + + + + +def get_assignment_group_id(assignment_group_name, course, create_if_necessary=False): + """ + gets the ID number of an assignment group from its name-as-string. + + `create_if_necessary`: There are two distinct behaviours available: + + False: [default] If such a group doesn't exist, this will raise. + True: Will make such an assignment group if it doesn't exist. + + Gods, I hope the preceding description made you feel like "well duh" because my names were that spot-on. If not, let's grab a beer together and talk about it. If you read this, you're amazing, and I'm glad you're using my software. I'm trying so hard to leave positive legacy! + """ + + existing_groups = course.get_assignment_groups() + + if not isinstance(assignment_group_name,str): + raise RuntimeError(f'assignment_group_name must be a string, but I got {assignment_group_name} of type {type(assignment_group_name)}') + + + for g in existing_groups: + if g.name == assignment_group_name: + return g.id + + + + if create_if_necessary: + msg = f'making new assignment group `{assignment_group_name}`' + logger.info(msg) + + group = course.create_assignment_group(name=assignment_group_name) + group.edit(name=assignment_group_name) # this feels stupid. didn't i just request its name be this? + + return group.id + else: + raise DoesntExist(f'cannot get assignment group id because an assignment group of name {assignment_group_name} does not already exist, and `create_if_necessary` is set to False') + + + + + +def create_or_get_assignment(name, course, even_if_exists = False): + """ + gets the `canvasapi.Assignment`. Can tell it to make the assignment if it didn't exist. + """ + + if is_assignment_already_uploaded(name,course): + if even_if_exists: + return find_assignment_in_course(name,course) + else: + raise AlreadyExists(f"assignment {name} already exists") + else: + # make new assignment of name in course. + return course.create_assignment(assignment={'name':name}) + + + +def create_or_get_page(name, course, even_if_exists): + """ + gets the `canvasapi.Page`. Can tell it to make the page if it didn't exist. + """ + + if is_page_already_uploaded(name,course): + + if even_if_exists: + return find_page_in_course(name,course) + else: + raise AlreadyExists(f"page {name} already exists") + else: + # make new assignment of name in course. + result = course.create_page(wiki_page={'body':"empty page",'title':name}) + return result + + + + +def create_or_get_module(module_name, course): + """ + gets the `canvasapi.Module`. Can tell it to make the module if it didn't exist. + """ + + try: + return get_module(module_name, course) + except DoesntExist as e: + return course.create_module(module={'name':module_name}) + + + + +def get_module(module_name, course): + """ + returns + * canvasapi.Module if such a module exists, + * raises if not + """ + modules = course.get_modules() + + for m in modules: + if m.name == module_name: + return m + + raise DoesntExist(f"tried to get module {module_name}, but it doesn't exist in the course") + + +def get_subfolder_named(folder, subfolder_name): + """ + gets the `canvasapi.Folder` with matching name. + + this is likely broken if subfolder has a / in its name, / gets converted to something else by Canvas. don't use / in subfolder names, that's not allowed + + raises if doesn't exist. + """ + + assert '/' not in subfolder_name, "this is likely broken if subfolder has a / in its name, / gets converted to something else by Canvas. don't use / in subfolder names, that's not allowed" + + current_subfolders = folder.get_folders() + for f in current_subfolders: + if f.name == subfolder_name: + return f + + raise DoesntExist(f'a subfolder of {folder.name} named {subfolder_name} does not currently exist') + + +def delete_module(module_name, course, even_if_exists): + ''' + Deletes a module by name-as-string. + ''' + + if even_if_exists: + try: + m = get_module(module_name, course) + m.delete() + except DoesntExist as e: + return + + else: + # this path is expected to raise if the module doesn't exist + m = get_module(module_name, course) + m.delete() + + diff --git a/markdown2canvas/exception.py b/markdown2canvas/exception.py new file mode 100644 index 0000000..ef6a246 --- /dev/null +++ b/markdown2canvas/exception.py @@ -0,0 +1,45 @@ +''' +Exception types emitted by markdown2canvas +''' + +__all__ = [ + 'AlreadyExists', + 'SetupError', + 'DoesntExist' +] + +class AlreadyExists(Exception): + """ + Used to indicate that you're trying to do a thing cautiously, and the thing already existed on Canvas. + """ + + def __init__(self, message, errors=""): + super().__init__(message) + + self.errors = errors + +class SetupError(Exception): + """ + Used to indicate that markdown2canvas couldn't get off the ground, or there's something else wrong that's not content-related but meta or config. + """ + + def __init__(self, message, errors=""): + super().__init__(message) + + self.errors = errors + + + + +class DoesntExist(Exception): + """ + Used when getting a thing, but it doesn't exist. + """ + + def __init__(self, message, errors=""): + super().__init__(message) + + self.errors = errors + + + diff --git a/markdown2canvas/exception/__init__.py b/markdown2canvas/exception/__init__.py deleted file mode 100644 index 209c61b..0000000 --- a/markdown2canvas/exception/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -class AlreadyExists(Exception): - - def __init__(self, message, errors=""): - # Call the base class constructor with the parameters it needs - super().__init__(message) - - self.errors = errors - -class SetupError(Exception): - - def __init__(self, message, errors=""): - # Call the base class constructor with the parameters it needs - super().__init__(message) - - self.errors = errors - - - - -class DoesntExist(Exception): - """ - Used when getting a thing, but it doesn't exist - """ - - def __init__(self, message, errors=""): - # Call the base class constructor with the parameters it needs - super().__init__(message) - - self.errors = errors \ No newline at end of file diff --git a/markdown2canvas/logging.py b/markdown2canvas/logging.py new file mode 100644 index 0000000..ff7403a --- /dev/null +++ b/markdown2canvas/logging.py @@ -0,0 +1,46 @@ +''' +Logging utilities. Uses the built-in `logging` library. This part could probably be improved to allow the user to set their own levels or turn on/off logging more easily. +''' + +__all__ = [ + 'today', 'log_dir', 'logger', 'file_handler' + ] + +import os.path as path +import os + +import logging + +# logging.basicConfig(encoding='utf-8') + +import datetime +today = datetime.datetime.today().strftime("%Y-%m-%d") + +log_level=logging.DEBUG + +log_dir = path.join(path.normpath(os.getcwd()), '_logs') + +if not path.exists(log_dir): + os.mkdir(log_dir) + +log_filename = path.join(log_dir, f'markdown2canvas_{today}.log') + + +log_encoding = 'utf-8' + +# make a logger object. we'll getLogger in the other files as needed. + +logger = logging.getLogger() # make a root-level logger using the defaulted options. see https://stackoverflow.com/questions/50714316/how-to-use-logging-getlogger-name-in-multiple-modules + +# adjust the logger for THIS module +logger.setLevel(log_level) + +# make a file handler and attach +file_handler = logging.FileHandler(log_filename, 'a', log_encoding) +logger.addHandler(file_handler) + +# a few messages to start +logging.debug(f'starting logging at {datetime.datetime.now()}') +logging.debug(f'reducing logging level of `requests` and `canvasapi` to WARNING') +logging.getLogger('canvasapi.requester').setLevel(logging.WARNING) +logging.getLogger('requests').setLevel(logging.WARNING) \ No newline at end of file diff --git a/markdown2canvas/setup_functions.py b/markdown2canvas/setup_functions.py new file mode 100644 index 0000000..cf442e2 --- /dev/null +++ b/markdown2canvas/setup_functions.py @@ -0,0 +1,68 @@ +''' +Functions for making a `canvasapi.Canvas` object with which to work + +Uses environment variables to let you specify things. +''' + +__all__ = ['get_canvas_key_url', 'make_canvas_api_obj'] + + +import os.path as path +import os + +import logging +logger = logging.getLogger(__name__) + +import canvasapi + + + + + +def get_canvas_key_url(): + """ + reads a file using an environment variable, namely the file specified in `CANVAS_CREDENTIAL_FILE`. + + We need the + + * API_KEY + * API_URL + + variables from that file. + """ + from os import environ + + cred_loc = environ.get('CANVAS_CREDENTIAL_FILE') + if cred_loc is None: + raise SetupError('`get_canvas_key_url()` needs an environment variable `CANVAS_CREDENTIAL_FILE`, containing the full path of the file containing your Canvas API_KEY, *including the file name*') + + # yes, this is scary. it was also low-hanging fruit, and doing it another way was going to be too much work + with open(path.join(cred_loc),encoding='utf-8') as cred_file: + exec(cred_file.read(),locals()) + + if isinstance(locals()['API_KEY'], str): + logger.info(f'using canvas with API_KEY as defined in {cred_loc}') + else: + raise SetupError(f'failing to use canvas. Make sure that file {cred_loc} contains a line of code defining a string variable `API_KEY="keyhere"`') + + return locals()['API_KEY'],locals()['API_URL'] + + +def make_canvas_api_obj(url=None): + """ + - reads the key from a python file, path to which must be in environment variable CANVAS_CREDENTIAL_FILE. + - optionally, pass in a url to use, in case you don't want the default one you put in your CANVAS_CREDENTIAL_FILE. + """ + + key, default_url = get_canvas_key_url() + + if not url: + url = default_url + + return canvasapi.Canvas(url, key) + + + + + + diff --git a/markdown2canvas/tool/__init__.py b/markdown2canvas/tool.py similarity index 89% rename from markdown2canvas/tool/__init__.py rename to markdown2canvas/tool.py index e39909f..2d3af9e 100644 --- a/markdown2canvas/tool/__init__.py +++ b/markdown2canvas/tool.py @@ -1,4 +1,8 @@ -import markdown2canvas as mc +""" +Provides a base class from which to derive when writing tools that will interact with `markdown2canvas` or `canvasapi`. +""" + +__all__ = ['Tool'] class Tool(object): @@ -59,7 +63,9 @@ def __init__(self, config_name = 'config.json'): def _read_config(self, config_name): - + """ + reads `config_name` (config.json by default), and unpacks `course_id`. Stores the de-serialized json file in self.config. + """ import json with open(config_name,'r') as f: config = json.load(f) diff --git a/markdown2canvas/free_functions/__init__.py b/markdown2canvas/translation_functions.py similarity index 57% rename from markdown2canvas/free_functions/__init__.py rename to markdown2canvas/translation_functions.py index 45ccd04..a515fae 100644 --- a/markdown2canvas/free_functions/__init__.py +++ b/markdown2canvas/translation_functions.py @@ -1,136 +1,27 @@ -import os.path as path -import os - -import logging -logger = logging.getLogger(__name__) - -import canvasapi - -def is_file_already_uploaded(filename,course): - """ - returns a boolean, true if there's a file of `filename` already in `course`. - - This function wants the full path to the file. - """ - return ( not find_file_in_course(filename,course) is None ) - - - - -def find_file_in_course(filename,course): - """ - Checks to see of the file at `filename` is already in the "files" part of `course`. - - It tests filename and size as reported on disk. If it finds a match, then it's up. - - This function wants the full path to the file. - """ - import os - - base = path.split(filename)[1] - - files = course.get_files() - for f in files: - if f.filename==base and f.size == path.getsize(filename): - return f - - return None - - - - - -def is_page_already_uploaded(name,course): - """ - returns a boolean indicating whether a page of the given `name` is already in the `course`. - """ - return ( not find_page_in_course(name,course) is None ) - - -def find_page_in_course(name,course): - """ - Checks to see if there's already a page named `name` as part of `course`. - - tests merely based on the name. assumes assignments are uniquely named. - """ - import os - pages = course.get_pages() - for p in pages: - if p.title == name: - return p - - return None - - - -def is_assignment_already_uploaded(name,course): - """ - returns a boolean indicating whether an assignment of the given `name` is already in the `course`. - """ - return ( not find_assignment_in_course(name,course) is None ) - - -def find_assignment_in_course(name,course): - """ - Checks to see if there's already an assignment named `name` as part of `course`. - - tests merely based on the name. assumes assingments are uniquely named. - """ - import os - assignments = course.get_assignments() - for a in assignments: - - if a.name == name: - return a - - return None - - - - -def get_canvas_key_url(): - """ - reads a file using an environment variable, namely the file specified in `CANVAS_CREDENTIAL_FILE`. - - We need the - - * API_KEY - * API_URL - - variables from that file. - """ - from os import environ - - cred_loc = environ.get('CANVAS_CREDENTIAL_FILE') - if cred_loc is None: - raise SetupError('`get_canvas_key_url()` needs an environment variable `CANVAS_CREDENTIAL_FILE`, containing the full path of the file containing your Canvas API_KEY, *including the file name*') - - # yes, this is scary. it was also low-hanging fruit, and doing it another way was going to be too much work - with open(path.join(cred_loc),encoding='utf-8') as cred_file: - exec(cred_file.read(),locals()) - - if isinstance(locals()['API_KEY'], str): - logger.info(f'using canvas with API_KEY as defined in {cred_loc}') - else: - raise SetupError(f'failing to use canvas. Make sure that file {cred_loc} contains a line of code defining a string variable `API_KEY="keyhere"`') - - return locals()['API_KEY'],locals()['API_URL'] - - -def make_canvas_api_obj(url=None): - """ - - reads the key from a python file, path to which must be in environment variable CANVAS_CREDENTIAL_FILE. - - optionally, pass in a url to use, in case you don't want the default one you put in your CANVAS_CREDENTIAL_FILE. - """ - - key, default_url = get_canvas_key_url() - - if not url: - url = default_url - - return canvasapi.Canvas(url, key) +""" +Functions for translating markdown to html, +putting headers/footers around content, +and manipulating links to link to or embed images and files on Canvas. +""" + +__all__ = [ + 'generate_course_link', + 'find_in_containing_directory_path', + 'preprocess_replacements', + 'preprocess_markdown_images', + 'get_default_property', + 'get_default_style_name', + 'get_default_replacements_name', + 'apply_style_markdown', + 'apply_style_html', + 'markdown2html', + 'adjust_html_for_images', + 'adjust_html_for_files' +] +import os.path as path +import os def generate_course_link(type,name,all_of_type,courseid=None): ''' @@ -367,26 +258,6 @@ def markdown2html(filename, course, replacements_path): -def find_local_images(html): - """ - constructs a map of local url's : Images - """ - from bs4 import BeautifulSoup - - soup = BeautifulSoup(html,features="lxml") - - local_images = {} - - all_imgs = soup.findAll("img") - - if all_imgs: - for img in all_imgs: - src = img["src"] - if src[:7] not in ['https:/','http://']: - local_images[src] = Image(path.abspath(src)) - - return local_images - @@ -424,26 +295,6 @@ def adjust_html_for_images(html, published_images, courseid): -def find_local_files(html): - """ - constructs a list of BareFiles, so that they can later be replaced with a url to a canvas thing - """ - from bs4 import BeautifulSoup - - soup = BeautifulSoup(html,features="lxml") - - local_files = {} - - all_links = soup.findAll("a") - - if all_links: - for file in all_links: - href = file["href"] - if path.exists(path.abspath(href)): - local_files[href] = BareFile(path.abspath(href)) - - return local_files - def adjust_html_for_files(html, published_files, courseid): @@ -475,123 +326,7 @@ def adjust_html_for_files(html, published_files, courseid): -def get_root_folder(course): - for f in course.get_folders(): - if f.full_name == 'course files': - return f - - - - - - - - -def get_assignment_group_id(assignment_group_name, course, create_if_necessary=False): - existing_groups = course.get_assignment_groups() - if not isinstance(assignment_group_name,str): - raise RuntimeError(f'assignment_group_name must be a string, but I got {assignment_group_name} of type {type(assignment_group_name)}') - - for g in existing_groups: - if g.name == assignment_group_name: - return g.id - - - - if create_if_necessary: - msg = f'making new assignment group `{assignment_group_name}`' - logger.info(msg) - - group = course.create_assignment_group(name=assignment_group_name) - group.edit(name=assignment_group_name) # this feels stupid. didn't i just request its name be this? - - return group.id - else: - raise DoesntExist(f'cannot get assignment group id because an assignment group of name {assignment_group_name} does not already exist, and `create_if_necessary` is set to False') - - - - - -def create_or_get_assignment(name, course, even_if_exists = False): - - if is_assignment_already_uploaded(name,course): - if even_if_exists: - return find_assignment_in_course(name,course) - else: - raise AlreadyExists(f"assignment {name} already exists") - else: - # make new assignment of name in course. - return course.create_assignment(assignment={'name':name}) - - - -def create_or_get_page(name, course, even_if_exists): - if is_page_already_uploaded(name,course): - - if even_if_exists: - return find_page_in_course(name,course) - else: - raise AlreadyExists(f"page {name} already exists") - else: - # make new assignment of name in course. - result = course.create_page(wiki_page={'body':"empty page",'title':name}) - return result - - - - -def create_or_get_module(module_name, course): - - try: - return get_module(module_name, course) - except DoesntExist as e: - return course.create_module(module={'name':module_name}) - - - - -def get_module(module_name, course): - """ - returns - * Module if such a module exists, - * raises if not - """ - modules = course.get_modules() - - for m in modules: - if m.name == module_name: - return m - - raise DoesntExist(f"tried to get module {module_name}, but it doesn't exist in the course") - - -def get_subfolder_named(folder, subfolder_name): - - assert '/' not in subfolder_name, "this is likely broken if subfolder has a / in its name, / gets converted to something else by Canvas. don't use / in subfolder names, that's not allowed" - - current_subfolders = folder.get_folders() - for f in current_subfolders: - if f.name == subfolder_name: - return f - - raise DoesntExist(f'a subfolder of {folder.name} named {subfolder_name} does not currently exist') - - -def delete_module(module_name, course, even_if_exists): - - if even_if_exists: - try: - m = get_module(module_name, course) - m.delete() - except DoesntExist as e: - return - - else: - # this path is expected to raise if the module doesn't exist - m = get_module(module_name, course) - m.delete() diff --git a/test/course_id.py b/test/course_id.py index 9586ff5..7c6fdd2 100644 --- a/test/course_id.py +++ b/test/course_id.py @@ -1 +1 @@ -test_course_id = 537006 +test_course_id = 705022 diff --git a/test/test_assignment.py b/test/test_assignment.py index 41c8e83..5b616b3 100644 --- a/test/test_assignment.py +++ b/test/test_assignment.py @@ -64,7 +64,7 @@ def test_already_online_raises(self, course, assignment): assignment.publish(course,overwrite=True) # the second publish, with overwrite=False, should raise - with pytest.raises(mc.AlreadyExists): + with pytest.raises(mc.exception.AlreadyExists): assignment.publish(course,overwrite=False) # default is False def test_doesnt_find_deleted(self, course, assignment): diff --git a/test/test_download_pages_to_markdown.py b/test/test_download_pages_to_markdown.py index cd68c8c..1d3ccd5 100644 --- a/test/test_download_pages_to_markdown.py +++ b/test/test_download_pages_to_markdown.py @@ -32,7 +32,7 @@ def test_aaa_can_download_all_pages(self): if os.path.exists(destination): shutil.rmtree(destination) - mc.download_pages(destination, self.course, even_if_exists=False) + mc.canvas2markdown.download_pages(destination, self.course, even_if_exists=False) def test_aaa_can_download_some_pages(self): import os, shutil @@ -41,7 +41,7 @@ def test_aaa_can_download_some_pages(self): shutil.rmtree(destination) my_filter = lambda title: 'test' in title.lower() - mc.download_pages(destination, self.course, even_if_exists=False, name_filter=my_filter) + mc.canvas2markdown.download_pages(destination, self.course, even_if_exists=False, name_filter=my_filter) if __name__ == '__main__': diff --git a/test/test_droplets.py b/test/test_droplets.py index 818cef3..ca69255 100644 --- a/test/test_droplets.py +++ b/test/test_droplets.py @@ -48,7 +48,7 @@ def test_already_online_raises(self, course, page): page.publish(course,overwrite=True) # the second publish, with overwrite=False, should raise - with pytest.raises(mc.AlreadyExists): + with pytest.raises(mc.exception.AlreadyExists): page.publish(course,overwrite=False) # default is False diff --git a/test/test_file.py b/test/test_file.py index e32a7da..0c8fd84 100644 --- a/test/test_file.py +++ b/test/test_file.py @@ -42,7 +42,7 @@ def test_already_online_raises(self, course, content): content.publish(course,overwrite=True) # the second publish, with overwrite=False, should raise - with pytest.raises(mc.AlreadyExists): + with pytest.raises(mc.exception.AlreadyExists): content.publish(course,overwrite=False) # default is False diff --git a/test/test_link.py b/test/test_link.py index 05058b0..49b4073 100644 --- a/test/test_link.py +++ b/test/test_link.py @@ -51,7 +51,7 @@ def test_already_online_raises(self,course,link): link.publish(course,overwrite=True) # the second publish, with overwrite=False, should raise - with pytest.raises(mc.AlreadyExists): + with pytest.raises(mc.exception.AlreadyExists): link.publish(course,overwrite=False) # default is False diff --git a/test/test_link_to_local_file.py b/test/test_link_to_local_file.py index d910e99..4dbb790 100644 --- a/test/test_link_to_local_file.py +++ b/test/test_link_to_local_file.py @@ -49,7 +49,7 @@ def test_already_online_raises(self, course, page): page.publish(course,overwrite=True) # the second publish, with overwrite=False, should raise - with pytest.raises(mc.AlreadyExists): + with pytest.raises(mc.exception.AlreadyExists): page.publish(course,overwrite=False) # default is False diff --git a/test/test_page.py b/test/test_page.py index 19fa8b7..58bcdaa 100644 --- a/test/test_page.py +++ b/test/test_page.py @@ -38,7 +38,7 @@ def test_already_online_raises(self, course, page_has_local_images): page_has_local_images.publish(course,overwrite=True) # the second publish, with overwrite=False, should raise - with pytest.raises(mc.AlreadyExists): + with pytest.raises(mc.exception.AlreadyExists): page_has_local_images.publish(course,overwrite=False) # default is False def test_doesnt_find_deleted(self, course, page_has_local_images): diff --git a/test/test_page_in_module.py b/test/test_page_in_module.py index d48152b..80543de 100644 --- a/test/test_page_in_module.py +++ b/test/test_page_in_module.py @@ -54,7 +54,7 @@ def test_already_online_raises(self, course, page_plain_text_in_a_module): page_plain_text_in_a_module.publish(course,overwrite=True) # the second publish, with overwrite=False, should raise - with pytest.raises(mc.AlreadyExists): + with pytest.raises(mc.exception.AlreadyExists): page_plain_text_in_a_module.publish(course,overwrite=False) # default is False diff --git a/test/test_style.py b/test/test_style.py index 8ad8bd0..a1b5e22 100644 --- a/test/test_style.py +++ b/test/test_style.py @@ -39,7 +39,7 @@ def test_already_online_raises(self, course, page_uses_droplets_via_style): page_uses_droplets_via_style.publish(course,overwrite=True) # the second publish, with overwrite=False, should raise - with pytest.raises(mc.AlreadyExists): + with pytest.raises(mc.exception.AlreadyExists): page_uses_droplets_via_style.publish(course,overwrite=False) # default is False From 9b777fd3e3e8e563b30ff9de01d54d32d176b4f2 Mon Sep 17 00:00:00 2001 From: Allison Beemer <66966224+anbeemer@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:22:24 -0500 Subject: [PATCH 15/57] change from even_if_exists to even_if_doesnt_exist --- test/test_page_in_module.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/test_page_in_module.py b/test/test_page_in_module.py index d48152b..6565fb2 100644 --- a/test/test_page_in_module.py +++ b/test/test_page_in_module.py @@ -31,11 +31,9 @@ def destination_modules(page_plain_text_in_a_module): yield page.metadata['modules'] -#self._delete_test_modules() - -def _delete_test_modules(self): - for m in self.destination_modules: - mc.delete_module(m, self.course, even_if_exists=True) +def _delete_test_modules(course, destination_modules): + for m in destination_modules: + mc.delete_module(m, course, even_if_doesnt_exist=True) @@ -64,12 +62,12 @@ def test_can_make_modules(self, course, destination_modules): def test_can_delete_modules(self, course, destination_modules): - + _delete_test_modules(course, destination_modules) for m in destination_modules: mc.create_or_get_module(m,course) for m in destination_modules: - mc.delete_module(m, course, even_if_exists=False) + mc.delete_module(m, course, even_if_doesnt_exist=False) From 32be4588dbe56ec33337fae565f6ec144100e9f1 Mon Sep 17 00:00:00 2001 From: Allison Beemer <66966224+anbeemer@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:23:10 -0500 Subject: [PATCH 16/57] Unit tests added to test_replacements --- test/test_replacements.py | 113 +++++++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 3 deletions(-) diff --git a/test/test_replacements.py b/test/test_replacements.py index d8c7381..be6b0f0 100644 --- a/test/test_replacements.py +++ b/test/test_replacements.py @@ -2,8 +2,11 @@ sys.path.insert(0,'../') import markdown2canvas as mc +import json + import pytest + @pytest.fixture(scope='class') def course(): import os @@ -17,21 +20,75 @@ def course(): @pytest.fixture(scope='class') -def page_using_defaults(course): +def page_using_defaults(): import os folder = 'uses_replacements_default' yield mc.Page(folder) + @pytest.fixture(scope='class') -def page_using_custom(course): +def page_using_custom(): import os folder = 'uses_replacements_custom' yield mc.Page(folder) +@pytest.fixture(scope='class') +def default_filename(): + with open('_course_metadata/defaults.json', "r", encoding="utf-8") as f: + defaults = f.read() + yield json.loads(defaults)['replacements'] + +@pytest.fixture(scope='class') +def replacements_default(default_filename): + with open(default_filename, "r", encoding="utf-8") as f: + yield f.read() + +@pytest.fixture(scope='class') +def uses_defaults_source(): + with open('uses_replacements_default/source.md', "r", encoding="utf-8") as f: + yield f.read() + +# @pytest.fixture(scope='class') +# def html_using_defaults(): +# with open('uses_replacements_default/result.html', "r", encoding="utf-8") as f: +# yield f.read() + +@pytest.fixture(scope='class') +def html_using_defaults(course): + a = course.get_pages(search_term = 'Test replacements using default replacements file')[0] + rev = a.show_latest_revision() + yield rev.body + + +@pytest.fixture(scope='class') +def replacements_custom(): + with open('_course_metadata/replacements2.json', "r", encoding="utf-8") as f: + yield f.read() + +@pytest.fixture(scope='class') +def uses_custom_source(): + with open('uses_replacements_custom/source.md', "r", encoding="utf-8") as f: + yield f.read() + +# @pytest.fixture(scope='class') +# def html_using_custom(): +# with open('uses_replacements_custom/result.html', "r", encoding="utf-8") as f: +# yield f.read() + +@pytest.fixture(scope='class') +def html_using_custom(course): + a = course.get_pages(search_term = 'Test replacements with custom replacements file')[0] + rev = a.show_latest_revision() + yield rev.body + + + + + class TestPage(): def test_can_publish(self, course, page_using_defaults, page_using_custom): @@ -39,7 +96,57 @@ def test_can_publish(self, course, page_using_defaults, page_using_custom): page_using_custom.publish(course,overwrite=True) - ##Removed a " as e_info" after the def in the following... doesn't seem to have hurt it? + def test_get_default_replacements_name(self): + path = mc.get_default_replacements_name() + assert path == '_course_metadata/replacements.json' + + + def test_removed_default(self, html_using_defaults, replacements_default, uses_defaults_source): + replacements_dict_default = json.loads(replacements_default) + for key in replacements_dict_default: + if key in uses_defaults_source: + assert key not in html_using_defaults + #Want to add something about the new thing being in the html + #assert replacements_dict_default[key] in html_using_defaults + + def test_replaced_default(self, html_using_defaults): + #default replacements that should translate seamlessly + assert 'with this text' in html_using_defaults + assert 'destination_without_spaces' in html_using_defaults + #check specific video options + assert '560' in html_using_defaults + assert '315' in html_using_defaults + assert 'https://www.youtube.com/embed/dQw4w9WgXcQ?si=BqTm4nbZOLTHaxnz' in html_using_defaults + assert 'YouTube video player' in html_using_defaults + assert 'accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share' in html_using_defaults + assert 'allowfullscreen' in html_using_defaults + + def test_removed_custom(self, html_using_custom, uses_custom_source, replacements_custom): + replacements_dict_custom = json.loads(replacements_custom) + for key in replacements_dict_custom: + if key in uses_custom_source: + assert key not in html_using_custom + assert replacements_dict_custom[key] in html_using_custom + + def test_replaced_custom(self, html_using_custom): + #custom replacements that should translate seamlessly + assert 'target custom replacement without space' in html_using_custom + assert 'target custom replacement from nospace' in html_using_custom + + def test_incorrect_replacement_custom(self, html_using_custom): + #First check that none of the default replacements show up in the custom replacements file + assert 'with this text' not in html_using_custom + assert 'destination_without_spaces' not in html_using_custom + assert 'https://www.youtube.com/embed/dQw4w9WgXcQ?si=BqTm4nbZOLTHaxnz' not in html_using_custom + + + def test_incorrect_replacement_default(self, html_using_defaults): + #First check that none of the default replacements show up in the custom replacements file + assert 'target custom replacement without space' not in html_using_defaults + assert 'target custom replacement from nospace' not in html_using_defaults + + + def test_missing_replacements(self): # constructing a page with a replacements file that doesn't exist should raise with pytest.raises(FileNotFoundError): From eda96085937539e28c654d4e41a06eac83ba9400 Mon Sep 17 00:00:00 2001 From: Allison Beemer <66966224+anbeemer@users.noreply.github.com> Date: Tue, 16 Jul 2024 09:58:55 -0500 Subject: [PATCH 17/57] Added note about replacement order --- docs/tutorials/text_replacements.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tutorials/text_replacements.rst b/docs/tutorials/text_replacements.rst index dab0155..e670aa3 100644 --- a/docs/tutorials/text_replacements.rst +++ b/docs/tutorials/text_replacements.rst @@ -25,6 +25,8 @@ Note that `_course_metadata/replacements.json` is just a regular old JSON file. Usage -------- +Keep in mind that the order of replacement is unspecified. Thus, it is important to choose keys that will not appear within values, and will not appear within source documents where replacement is undesired. + Custom text replacements per-content -------------------------------------- From 3e6e0120c99629dbe6f438a685571df208916d5a Mon Sep 17 00:00:00 2001 From: Allison Beemer <66966224+anbeemer@users.noreply.github.com> Date: Tue, 16 Jul 2024 11:34:55 -0500 Subject: [PATCH 18/57] Added custom style page to check --- test/_styles/custom/Ruapehu_and_Ngauruhoe.jpg | Bin 0 -> 553331 bytes test/_styles/custom/footer.html | 4 ++ test/_styles/custom/footer.md | 4 ++ test/_styles/custom/header.html | 7 ++++ test/_styles/custom/header.md | 4 ++ .../hauser_menagerie.jpg | Bin 0 -> 113107 bytes test/uses_droplets_via_style_custom/meta.json | 6 +++ test/uses_droplets_via_style_custom/source.md | 39 ++++++++++++++++++ 8 files changed, 64 insertions(+) create mode 100644 test/_styles/custom/Ruapehu_and_Ngauruhoe.jpg create mode 100644 test/_styles/custom/footer.html create mode 100644 test/_styles/custom/footer.md create mode 100644 test/_styles/custom/header.html create mode 100644 test/_styles/custom/header.md create mode 100644 test/uses_droplets_via_style_custom/hauser_menagerie.jpg create mode 100644 test/uses_droplets_via_style_custom/meta.json create mode 100644 test/uses_droplets_via_style_custom/source.md diff --git a/test/_styles/custom/Ruapehu_and_Ngauruhoe.jpg b/test/_styles/custom/Ruapehu_and_Ngauruhoe.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f24a8861b1eb5d0bb2cc45425234154580c8a624 GIT binary patch literal 553331 zcmb@ud03KL`!