From cf6f36d2fe71f42430466922bafa3086319ce998 Mon Sep 17 00:00:00 2001 From: "David J. Malan" Date: Wed, 22 Nov 2017 13:44:21 -0500 Subject: [PATCH 1/4] added printing of GitHub response for auth failures to help debugging in verbose mode --- setup.py | 2 +- submit50.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ba0731b..a5cc9eb 100644 --- a/setup.py +++ b/setup.py @@ -79,5 +79,5 @@ class CustomInstall(install): }, data_files=create_mo_files(), url="https://github.com/cs50/submit50", - version="2.4.6" + version="2.4.7" ) diff --git a/submit50.py b/submit50.py index 03355ea..2f64abc 100755 --- a/submit50.py +++ b/submit50.py @@ -220,10 +220,16 @@ def rlinput(prompt, prefill=""): # check if incorrect password if res.status_code == 401: + if run.verbose: + cprint(str(res.headers), "yellow") + cprint(res.text, "yellow") raise Error(_("Invalid username and/or password.")) # check for other error elif res.status_code != 200: + if run.verbose: + cprint(str(res.headers), "yellow") + cprint(res.text, "yellow") raise Error(_("Could not authenticate user.")) # canonicalize (capitalization of) username, From a58ae11137e09ca2e66554c5acd23731eeb7eb8c Mon Sep 17 00:00:00 2001 From: "David J. Malan" Date: Wed, 3 Jan 2018 14:52:11 -0500 Subject: [PATCH 2/4] working on resolving symlinks --- setup.py | 2 +- submit50.py | 28 +++++++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index a5cc9eb..52d9dcf 100644 --- a/setup.py +++ b/setup.py @@ -79,5 +79,5 @@ class CustomInstall(install): }, data_files=create_mo_files(), url="https://github.com/cs50/submit50", - version="2.4.7" + version="2.4.8" ) diff --git a/submit50.py b/submit50.py index 2f64abc..a6a6f23 100755 --- a/submit50.py +++ b/submit50.py @@ -2,6 +2,7 @@ import argparse import atexit +import base64 import datetime import distutils import gettext @@ -453,24 +454,29 @@ def submit(org, branch): except ValueError: slug, src = branch, "cs50/checks" - # ensure slug exists - file, submit.EXCLUDE = tempfile.mkstemp() - url = "https://github.com/{}/raw/master/{}/submit50/exclude".format(src, slug) + # check for common mistakes + if slug.startswith("/") and slug.endswith("/"): + raise Error(_("Invalid slug. Did you mean {}, without the leading and trailing slashes?".format(slug.strip("/")))) + elif slug.startswith("/"): + raise Error(_("Invalid slug. Did you mean {}, without the leading slash?".format(slug.strip("/")))) + elif slug.endswith("/"): + raise Error(_("Invalid slug. Did you mean {}, without the trailing slash?".format(slug.strip("/")))) + + # get slug's gitignore file try: - urllib.request.urlretrieve(url, filename=submit.EXCLUDE) - lines = open(submit.EXCLUDE) + res = requests.get("https://api.github.com/repos/{}/contents/{}/submit50/exclude".format(src, slug), + headers={"Accept": "application/vnd.github.v3.raw"}) + assert res.status_code == 200 + fd, submit.EXCLUDE = tempfile.mkstemp() + with os.fdopen(fd, "w") as file: + file.write(res.text) + lines = res.text.splitlines() except Exception as e: if run.verbose: cprint(str(e)) e = Error(_("Invalid slug. Did you mean to submit something else?")) e.__cause__ = None raise e - if slug.startswith("/") and slug.endswith("/"): - raise Error(_("Invalid slug. Did you mean {}, without the leading and trailing slashes?".format(slug.strip("/")))) - elif slug.startswith("/"): - raise Error(_("Invalid slug. Did you mean {}, without the leading slash?".format(slug.strip("/")))) - elif slug.endswith("/"): - raise Error(_("Invalid slug. Did you mean {}, without the trailing slash?".format(slug.strip("/")))) # check for missing files missing = [] From b8ace99b5bf7fc91e4a86c98b121320b55a9ad8c Mon Sep 17 00:00:00 2001 From: "David J. Malan" Date: Wed, 3 Jan 2018 14:55:50 -0500 Subject: [PATCH 3/4] reverting --- setup.py | 2 +- submit50.py | 771 +++++----------------------------------------------- 2 files changed, 71 insertions(+), 702 deletions(-) diff --git a/setup.py b/setup.py index 52d9dcf..a5cc9eb 100644 --- a/setup.py +++ b/setup.py @@ -79,5 +79,5 @@ class CustomInstall(install): }, data_files=create_mo_files(), url="https://github.com/cs50/submit50", - version="2.4.8" + version="2.4.7" ) diff --git a/submit50.py b/submit50.py index a6a6f23..a5cc9eb 100755 --- a/submit50.py +++ b/submit50.py @@ -1,714 +1,83 @@ -from __future__ import print_function - -import argparse -import atexit -import base64 -import datetime -import distutils -import gettext -import itertools -import json -import os -import pexpect -import pipes -import platform -import re -import readline -import requests -import select -import shlex -import shutil -import signal -import subprocess -import sys -import termcolor -import tempfile -import textwrap -import time -import traceback - -from backports.shutil_get_terminal_size import get_terminal_size -from backports.shutil_which import which -from distutils.spawn import find_executable -from distutils.version import StrictVersion -from pkg_resources import get_distribution, parse_version -from six.moves import urllib -from threading import Thread - -# Internationalization -gettext.bindtextdomain("messages", os.path.join(sys.prefix, "submit50/locale")) -gettext.textdomain("messages") -_ = gettext.gettext - -# globals -# require python 2.7+ -if sys.version_info < (2, 7): - sys.exit(_("You have an old version of python. Install version 2.7 or higher.")) -if sys.version_info < (3, 0): - input = raw_input -if not hasattr(shlex, "quote"): - shlex.quote = pipes.quote - -timestamp = None - - -class Error(Exception): - """Exception raised for errors.""" - pass +from glob import glob +from os import walk +from os.path import isfile, join, splitext +from setuptools import setup +from setuptools.command.develop import develop +from setuptools.command.install import install +from subprocess import call +from sys import platform, version_info + +def create_mo_files(): + """Compiles .po files in local/LANG to .mo files and returns them as array of data_files""" + + mo_files=[] + for prefix in glob("locale/*"): + for _,_,files in walk(prefix): + for file in files: + if file.endswith(".po"): + po_file = join(prefix, file) + mo_file = splitext(po_file)[0] + ".mo" + call(["msgfmt", "-o", mo_file, po_file]) + mo_files.append((join("submit50", prefix, "LC_MESSAGES"), [mo_file])) + return mo_files -class _Getch: +def install_certs(cmd): """ - Get a single character from standard input. + Decorator for classes subclassing one of setuptools commands. - https://stackoverflow.com/a/510364 + Installs certificates before installing the package when running + Python >= 3.6 on Mac OS. """ + orig_run = cmd.run - class _GetchUnix: - def __init__(self): - import tty - import sys - - def __call__(self): - import sys - import termios - import tty - fd = sys.stdin.fileno() - old_settings = termios.tcgetattr(fd) - try: - tty.setraw(sys.stdin.fileno()) - ch = sys.stdin.read(1) - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) - return ch - - class _GetchWindows: - def __init__(self): - import msvcrt - - def __call__(self): - import msvcrt - return msvcrt.getch() - - def __init__(self): - try: - self.impl = _Getch._GetchWindows() - except ImportError: - self.impl = _Getch._GetchUnix() - - def __call__(self): - return self.impl() - - -getch = _Getch() - - -# submit50 -def main(): - - # listen for ctrl-c - signal.signal(signal.SIGINT, handler) - - # clean up on normal exit - atexit.register(teardown) - - # define command-line arguments - parser = argparse.ArgumentParser() - parser.add_argument("-v", "--verbose", - action="store_true", - help=_("show commands being executed")) - parser.add_argument("slug", help=_("prescribed identifier of work to submit")) - args = vars(parser.parse_args()) - - # submit50 -v - # submit50 --verbose - if args["verbose"]: - run.verbose = True - - # submit50 slug - submit("submit50", args["slug"]) - - # kthxbai - sys.exit(0) - - -def authenticate(org): - """Authenticate user.""" - - # cache credentials in ~/.git-credential-cache/:org - cache = os.path.expanduser("~/.git-credential-cache") - try: - os.mkdir(cache, 0o700) - except BaseException: - pass - authenticate.SOCKET = os.path.join(cache, "submit50") - - spawn = pexpect.spawn if sys.version_info < (3, 0) else pexpect.spawnu - child = spawn( - "git -c credential.helper='cache --socket {}' credential fill".format(authenticate.SOCKET)) - child.sendline("") - if child.expect(["Username:", "Password:", pexpect.EOF]) == 2: - clear_credentials() - username, password = re.search( - "username=([^\r]+)\r\npassword=([^\r]+)", child.before, re.MULTILINE).groups() - else: - try: - username = run("git config --global credential.https://github.com/submit50.username", quiet=True) - except Error: - username = None - password = None - child.close() - - def rlinput(prompt, prefill=""): - """ - Input function that uses a prefill value and advanced line editing. - - https://stackoverflow.com/a/2533142 - """ - readline.set_startup_hook(lambda: readline.insert_text(prefill)) - try: - return input(prompt) - finally: - readline.set_startup_hook() - - # prompt for credentials - progress(False) # because not using cprint herein - if not password: - - # prompt for username, prefilling if possible - progress(False) - try: - username = rlinput(_("GitHub username: "), username).strip() - except EOFError: - print() - - # prompt for password - print(_("GitHub password: "), end="") - sys.stdout.flush() - password = str() - while True: - ch = getch() - if ch in ["\n", "\r"]: # Enter - print() - break - elif ch == "\177": # DEL - if len(password) > 0: - password = password[:-1] - print("\b \b", end="") - sys.stdout.flush() - elif ch == "\3": # ctrl-c - print("^C", end="") - os.kill(os.getpid(), signal.SIGINT) - elif ch == "\4": # ctrl-d - print() - break - else: - password += ch - print("*", end="") - sys.stdout.flush() - - # authenticate user - email = "{}@users.noreply.github.com".format(username) - res = requests.get("https://api.github.com/user", - auth=(username, password)) - - # check for 2-factor authentication - # http://github3.readthedocs.io/en/develop/examples/oauth.html?highlight=token - if "X-GitHub-OTP" in res.headers: - password = two_factor(org, username, password) - res = requests.get("https://api.github.com/user", - auth=(username, password)) - - # check if incorrect password - if res.status_code == 401: - if run.verbose: - cprint(str(res.headers), "yellow") - cprint(res.text, "yellow") - raise Error(_("Invalid username and/or password.")) - - # check for other error - elif res.status_code != 200: - if run.verbose: - cprint(str(res.headers), "yellow") - cprint(res.text, "yellow") - raise Error(_("Could not authenticate user.")) - - # canonicalize (capitalization of) username, - # especially if user logged in via email address - username = res.json()["login"] - - # cache credentials for 1 week - timeout = int(datetime.timedelta(weeks=1).total_seconds()) - run("git -c credential.helper='cache --socket {} --timeout {}' " - "-c credentialcache.ignoresighup=true credential approve".format(authenticate.SOCKET, timeout), - lines=["username={}".format(username), "password={}".format(password), "", ""], - quiet=True) - - # return credentials - return (username, password, email) - - -authenticate.SOCKET = None - - -def clear_credentials(): - """Clear git credential cache """ - run("git credential-cache --socket {} exit".format(authenticate.SOCKET)) - if platform.system() == "Darwin": - try: - run("git credential-osxkeychain erase", lines=["host=github.com", "protocol=https", ""]) - except Error: - pass - - -def cprint(text="", color=None, on_color=None, attrs=None, **kwargs): - """Colorizes text (and wraps to terminal's width).""" - - # update progress - progress(False) - - # assume 80 in case not running in a terminal - columns, lines = get_terminal_size() - if columns == 0: - columns = 80 # because get_terminal_size's default fallback doesn't work in pipes - - # print text - termcolor.cprint(textwrap.fill(text, columns, drop_whitespace=False), - color=color, on_color=on_color, attrs=attrs, **kwargs) - - -def excepthook(type, value, tb): - """Report an exception.""" - excepthook.ignore = False - progress(False) - teardown() - if type is Error and str(value): - cprint(str(value), "yellow") - elif type is requests.exceptions.ConnectionError: - cprint(_("Could not connect to GitHub."), "yellow") - else: - if run.verbose: - traceback.print_exception(type, value, tb) - cprint(_("Sorry, something's wrong! Let sysadmins@cs50.harvard.edu know!"), "yellow") - if authenticate.SOCKET: # not set when using SSH - try: - clear_credentials() - except Exception: - pass - cprint(_("Submission cancelled."), "red") - - -sys.excepthook = excepthook - - -def handler(number, frame): - """Handle SIGINT.""" - os.system("stty sane 2> {}".format(os.devnull)) # in case signalled from input_with_prefill - if progress.progressing: - progress(False) - else: - cprint() - try: - clear_credentials() - except Exception: - pass - teardown() - cprint(_("{} cancelled.".format(handler.type.capitalize())), "red") - os._exit(0) - -handler.type = "submission" + def run(self): + if platform == "darwin" and version_info >= (3, 6): + INSTALL_CERTS = "/Applications/Python 3.6/Install Certificates.command" + if not isfile(INSTALL_CERTS) or call(INSTALL_CERTS) != 0: + raise RuntimeError("Error installing certificates.") + orig_run(self) -def run(command, cwd=None, env=None, lines=[], password=None, quiet=False, timeout=None): - """Run a command.""" + cmd.run = run + return cmd - # echo command - if run.verbose: - cprint(command, attrs=["bold"]) - # include GIT_DIR and GIT_WORK_TREE in env - if not env: - env = { - "GIT_DIR": run.GIT_DIR, - "GIT_WORK_TREE": run.GIT_WORK_TREE, - "HOME": os.path.expanduser("~") - } - if os.getenv("SSH_AGENT_PID"): - env["SSH_AGENT_PID"] = os.getenv("SSH_AGENT_PID") - if os.getenv("SSH_AUTH_SOCK"): - env["SSH_AUTH_SOCK"] = os.getenv("SSH_AUTH_SOCK") - - # spawn command - if sys.version_info < (3, 0): - child = pexpect.spawn(command, cwd=cwd, env=env, ignore_sighup=True, timeout=timeout) - else: - child = pexpect.spawnu( - command, - cwd=cwd, - encoding="utf-8", - env=env, - ignore_sighup=True, - timeout=timeout) - - # send output of command to stdout only if run with --verbose (and not quieted by caller) - if run.verbose and not quiet: - child.logfile_read = sys.stdout - - # wait for prompt, send password - if password: - i = child.expect(["Password for '.*': ", pexpect.EOF]) - if i == 0: - child.sendline(password) - - # send lines of input - for line in lines: - child.sendline(line) - - # read output, check status - command_output = child.read().strip() - child.close() - if child.signalstatus is None and child.exitstatus != 0: - raise Error() - return command_output - - -run.GIT_DIR = tempfile.mkdtemp() -run.GIT_WORK_TREE = os.getcwd() -run.verbose = False - - -def progress(message=""): - """Display a progress bar as dots.""" - - # don't show in verbose mode - if run.verbose: - if message: - print(message + "...") - return - - # stop progressing if already progressing - if progress.progressing: - progress.progressing = False - progress.thread.join() - sys.stdout.write("\n") - sys.stdout.flush() - - # display dots if message passed - if message: - def progress_helper(): - sys.stdout.write(message + "...") - sys.stdout.flush() - while progress.progressing: - sys.stdout.write(".") - sys.stdout.flush() - time.sleep(0.5) - progress.progressing = True - progress.thread = Thread(target=progress_helper) - progress.thread.start() - - -progress.progressing = False - - -def submit(org, branch): - """Submit work.""" - - # check announcements - res = requests.get("https://cs50.me/status/submit50") - if res.status_code == 200 and res.text.strip(): - raise Error(res.text.strip()) - - # require git 2.7+, so that credential-cache--daemon ignores SIGHUP - # https://github.com/git/git/blob/v2.7.0/credential-cache--daemon.c - if not which("git"): - raise Error(_("You don't have git. Install git, then re-run {}!".format(org))) - version = subprocess.check_output(["git", "--version"]).decode("utf-8") - matches = re.search(r"^git version (\d+\.\d+\.\d+).*$", version) - if not matches or StrictVersion(matches.group(1)) < StrictVersion("2.7.0"): - raise Error( - _("You have an old version of git. Install version 2.7 or later, then re-run {}!".format(org))) - - # update progress - progress("Connecting") - - # compute timestamp - global timestamp - headers = requests.get("https://api.github.com/").headers - timestamp = datetime.datetime.strptime(headers["Date"], "%a, %d %b %Y %H:%M:%S %Z") - timestamp = timestamp.strftime("%Y%m%dT%H%M%SZ") - - # check version - res = requests.get("https://cs50.me/versions/submit50") - if res.status_code != 200: - raise Error(_("You have an unknown version of submit50. " - "Email sysadmins@cs50.harvard.edu!")) - version_required = res.text.strip() - if parse_version(version_required) > parse_version(get_distribution("submit50").version): - raise Error(_("You have an old version of submit50. " - "Run update50, then re-run {}!".format(org))) - - # separate branch into slug and repo - check_repo = "@cs50/checks" - branch = branch if not branch.endswith(check_repo) else branch[:-len(check_repo)] - try: - slug, src = branch.split("@") - except ValueError: - slug, src = branch, "cs50/checks" - - # check for common mistakes - if slug.startswith("/") and slug.endswith("/"): - raise Error(_("Invalid slug. Did you mean {}, without the leading and trailing slashes?".format(slug.strip("/")))) - elif slug.startswith("/"): - raise Error(_("Invalid slug. Did you mean {}, without the leading slash?".format(slug.strip("/")))) - elif slug.endswith("/"): - raise Error(_("Invalid slug. Did you mean {}, without the trailing slash?".format(slug.strip("/")))) - - # get slug's gitignore file - try: - res = requests.get("https://api.github.com/repos/{}/contents/{}/submit50/exclude".format(src, slug), - headers={"Accept": "application/vnd.github.v3.raw"}) - assert res.status_code == 200 - fd, submit.EXCLUDE = tempfile.mkstemp() - with os.fdopen(fd, "w") as file: - file.write(res.text) - lines = res.text.splitlines() - except Exception as e: - if run.verbose: - cprint(str(e)) - e = Error(_("Invalid slug. Did you mean to submit something else?")) - e.__cause__ = None - raise e - - # check for missing files - missing = [] - for line in lines: - matches = re.match(r"^\s*#\s*([^\s]+)\s*$", line) - if matches: - pattern = matches.group(1) - if pattern[:-1] == "/": - if not os.path.isdir(pattern): - missing.append(pattern) - elif not os.path.isfile(pattern): - missing.append(pattern) - if missing: - cprint(_("You seem to be missing these files:")) - for pattern in missing: - cprint(" {}".format(pattern)) - raise Error(_("Ensure you have the required files before submitting.")) - - # update progress - progress(_("Authenticating")) - - # authenticate user via SSH - try: - - # require ssh - assert which("ssh") - - # require GitHub username in ~/.gitconfig - username, password = run( - "git config --global credential.https://github.com/submit50.username", quiet=True), None - email = "{}@users.noreply.github.com".format(username) - repo = "git@github.com:{}/{}.git".format(org, username) - progress(False) - - # require ssh-agent - child = pexpect.spawn("ssh git@github.com") - i = child.expect(["Enter passphrase for key", "Are you sure you want to continue connecting", pexpect.EOF]) - child.close() - assert i == 2 - - # authenticate user via HTTPS - except BaseException: - username, password, email = authenticate(org) - repo = "https://{}@github.com/{}/{}".format(username, org, username) - - # update progress - progress(_("Preparing")) - - # clone repository - try: - run("git clone --bare {} {}".format(shlex.quote(repo), - shlex.quote(run.GIT_DIR)), password=password) - except BaseException: - if password: - e = Error(_("Looks like {} isn't enabled for your account yet. " - "Go to https://cs50.me/authorize and make sure you accept any pending invitations!".format(org, org))) - else: - e = Error(_("Looks like you have the wrong username in ~/.gitconfig or {} isn't yet enabled for your account. " - "Double-check ~/.gitconfig and then log into https://cs50.me/ in a browser, " - "click \"Authorize application\" if prompted, and re-run {} here.".format(org, org))) - e.__cause__ = None - raise e - - # check out .gitattributes, if any, temporarily shadowing student's, if any - if os.path.isfile(".gitattributes"): - submit.ATTRIBUTES = ".gitattributes.{}".format(round(time.time())) - os.rename(".gitattributes", submit.ATTRIBUTES) - try: - run("git checkout --force {} .gitattributes".format(branch)) - except Exception: - pass - - # set options - tag = "{}@{}".format(branch, timestamp) - run("git config user.email {}".format(shlex.quote(email))) - run("git config user.name {}".format(shlex.quote(username))) - run("git symbolic-ref HEAD refs/heads/{}".format(shlex.quote(branch))) - - # patterns of file names to exclude - run("git config core.excludesFile {}".format(shlex.quote(submit.EXCLUDE))) - - # blocklist for git-lfs - # https://github.com/git-lfs/git-lfs/blob/master/commands/command_track.go - with open("{}/info/exclude".format(run.GIT_DIR), "w") as file: - file.write(".git*\n") - file.write(".lfs*\n") - - # adds, modifies, and removes index entries to match the working tree - run("git add --all") - - # get file lists - files = run("git ls-files").splitlines() - others = run("git ls-files --exclude-from={}/info/exclude --other".format(run.GIT_DIR)).splitlines() - - # unescape any octal codes in lists - # https://stackoverflow.com/a/46650050/5156190 - def unescape(s): - if s.startswith('"') and s.endswith('"'): - return ( - s.replace('"', '') - .encode("latin1") - .decode("unicode-escape") - .encode("latin1") - .decode("utf8") - ) - return s - files = [unescape(file) for file in files] - others = [unescape(other) for other in others] - - # hide .gitattributes, if any, from output - if ".gitattributes" in files: - files.remove(".gitattributes") - - # check for large files > 100 MB (and huge files > 2 GB) - # https://help.github.com/articles/conditions-for-large-files/ - # https://help.github.com/articles/about-git-large-file-storage/ - larges, huges = [], [] - for file in files: - size = os.path.getsize(file) - if size > (100 * 1024 * 1024): - larges.append(file) - elif size > (2 * 1024 * 1024 * 1024): - huges.append(file) - if len(huges) > 0: - raise Error(_("These files are too large to be submitted:\n{}\n" - "Remove these files from your directory " - "and then re-run {}!").format("\n".join(huges), org)) - elif len(larges) > 0: - if not which("git-lfs"): - raise Error(_("These files are too large to be submitted:\n{}\n" - "Install git-lfs (or remove these files from your directory) " - "and then re-run {}!").format("\n".join(larges), org)) - run("git lfs install --local") - run("git config credential.helper cache") # for pre-push hook - for large in larges: - run("git rm --cached {}".format(large)) - run("git lfs track {}".format(large)) - run("git add {}".format(large)) - run("git add --force .gitattributes") - - # files that will be submitted - if len(files) == 0: - raise Error(_("No files in this directory are expected for submission.")) - - # prompts for submit50 - if org == "submit50": - if files: - cprint(_("Files that will be submitted:"), "green") - for file in files: - cprint("./{}".format(file), "green") - - # files that won't be submitted - if others: - cprint(_("Files that won't be submitted:"), "yellow") - for other in others: - cprint("./{}".format(other), "yellow") - - # prompt for honesty - readline.clear_history() - try: - answer = input(_("Keeping in mind the course's policy on academic honesty, " - "are you sure you want to submit these files (yes/no)? ")) - except EOFError: - answer = None - print() - if not answer or not re.match("^\s*(?:y|yes)\s*$", answer, re.I): - raise Error(_("No files were submitted.")) - - # update progress - if org == "submit50": - progress(_("Submitting")) - else: - progress(_("Uploading")) - - # push branch - run("git commit --allow-empty --message='{}'".format(timestamp)) - commit_hash = run("git rev-parse HEAD") - run("git push origin 'refs/heads/{}'".format(branch), password=password) - - # successful submission - if org == "submit50": - cprint(_("Submitted {}! See https://cs50.me/submissions.").format(branch), - "green") - progress(False) - return username, commit_hash - - -submit.ATTRIBUTES = None -submit.EXCLUDE = None - - -def teardown(): - """Delete temporary directory and temporary file, restore any attributes.""" - if os.path.isfile(".gitattributes"): - try: - os.remove(".gitattributes") - except Exception: - pass - if submit.ATTRIBUTES: - try: - os.rename(submit.ATTRIBUTES, ".gitattributes") - except Exception: - pass - shutil.rmtree(run.GIT_DIR, ignore_errors=True) - if submit.EXCLUDE: - try: - os.remove(submit.EXCLUDE) - except Exception: - pass +@install_certs +class CustomDevelop(develop): + pass -def two_factor(org, username, password): - """Get one-time authentication code.""" - # send authentication request - requests.post("https://api.github.com/authorizations", - auth=(username, password)) - while True: - cprint("Authentication code:", end=" ") - sys.stdout.flush() - code = input() - if code: - break - data = json.dumps({"scopes": ["repo", "user"], "note": "{} {}".format(org, timestamp)}) - res = requests.post("https://api.github.com/authorizations", - auth=(username, password), - data=data, - headers={"X-GitHub-OTP": str(code)}) - if res.status_code == 201 and "token" in res.json(): - return res.json()["token"] - else: - raise Error(_("Could not complete two-factor authentication.")) +@install_certs +class CustomInstall(install): + pass -if __name__ == "__main__": - main() +setup( + author="CS50", + author_email="sysadmins@cs50.harvard.edu", + classifiers=[ + "Intended Audience :: Education", + "Programming Language :: Python :: 3", + "Topic :: Education", + "Topic :: Utilities" + ], + description="This is submit50, with which you can submit solutions to \ +problems for CS50.", + install_requires=[ + "backports.shutil_get_terminal_size", "backports.shutil_which", + "pexpect>=4.0", "requests", "six", "termcolor" + ], + keywords=["submit", "submit50"], + name="submit50", + py_modules=["submit50"], + cmdclass={ + "develop": CustomDevelop, + "install": CustomInstall + }, + entry_points={ + "console_scripts": ["submit50=submit50:main"] + }, + data_files=create_mo_files(), + url="https://github.com/cs50/submit50", + version="2.4.7" +) From 497454786885c4a7d77bc09d03dad37ca46022f2 Mon Sep 17 00:00:00 2001 From: "David J. Malan" Date: Wed, 3 Jan 2018 14:58:25 -0500 Subject: [PATCH 4/4] reverting --- submit50.py | 765 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 695 insertions(+), 70 deletions(-) diff --git a/submit50.py b/submit50.py index a5cc9eb..2f64abc 100755 --- a/submit50.py +++ b/submit50.py @@ -1,83 +1,708 @@ -from glob import glob -from os import walk -from os.path import isfile, join, splitext -from setuptools import setup -from setuptools.command.develop import develop -from setuptools.command.install import install -from subprocess import call -from sys import platform, version_info - -def create_mo_files(): - """Compiles .po files in local/LANG to .mo files and returns them as array of data_files""" - - mo_files=[] - for prefix in glob("locale/*"): - for _,_,files in walk(prefix): - for file in files: - if file.endswith(".po"): - po_file = join(prefix, file) - mo_file = splitext(po_file)[0] + ".mo" - call(["msgfmt", "-o", mo_file, po_file]) - mo_files.append((join("submit50", prefix, "LC_MESSAGES"), [mo_file])) +from __future__ import print_function + +import argparse +import atexit +import datetime +import distutils +import gettext +import itertools +import json +import os +import pexpect +import pipes +import platform +import re +import readline +import requests +import select +import shlex +import shutil +import signal +import subprocess +import sys +import termcolor +import tempfile +import textwrap +import time +import traceback + +from backports.shutil_get_terminal_size import get_terminal_size +from backports.shutil_which import which +from distutils.spawn import find_executable +from distutils.version import StrictVersion +from pkg_resources import get_distribution, parse_version +from six.moves import urllib +from threading import Thread + +# Internationalization +gettext.bindtextdomain("messages", os.path.join(sys.prefix, "submit50/locale")) +gettext.textdomain("messages") +_ = gettext.gettext + +# globals +# require python 2.7+ +if sys.version_info < (2, 7): + sys.exit(_("You have an old version of python. Install version 2.7 or higher.")) +if sys.version_info < (3, 0): + input = raw_input +if not hasattr(shlex, "quote"): + shlex.quote = pipes.quote + +timestamp = None + + +class Error(Exception): + """Exception raised for errors.""" + pass - return mo_files -def install_certs(cmd): +class _Getch: """ - Decorator for classes subclassing one of setuptools commands. + Get a single character from standard input. - Installs certificates before installing the package when running - Python >= 3.6 on Mac OS. + https://stackoverflow.com/a/510364 """ - orig_run = cmd.run - def run(self): - if platform == "darwin" and version_info >= (3, 6): - INSTALL_CERTS = "/Applications/Python 3.6/Install Certificates.command" - if not isfile(INSTALL_CERTS) or call(INSTALL_CERTS) != 0: - raise RuntimeError("Error installing certificates.") - orig_run(self) + class _GetchUnix: + def __init__(self): + import tty + import sys - cmd.run = run - return cmd + def __call__(self): + import sys + import termios + import tty + fd = sys.stdin.fileno() + old_settings = termios.tcgetattr(fd) + try: + tty.setraw(sys.stdin.fileno()) + ch = sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) + return ch + class _GetchWindows: + def __init__(self): + import msvcrt -@install_certs -class CustomDevelop(develop): - pass + def __call__(self): + import msvcrt + return msvcrt.getch() + def __init__(self): + try: + self.impl = _Getch._GetchWindows() + except ImportError: + self.impl = _Getch._GetchUnix() -@install_certs -class CustomInstall(install): - pass + def __call__(self): + return self.impl() + + +getch = _Getch() + + +# submit50 +def main(): + + # listen for ctrl-c + signal.signal(signal.SIGINT, handler) + + # clean up on normal exit + atexit.register(teardown) + + # define command-line arguments + parser = argparse.ArgumentParser() + parser.add_argument("-v", "--verbose", + action="store_true", + help=_("show commands being executed")) + parser.add_argument("slug", help=_("prescribed identifier of work to submit")) + args = vars(parser.parse_args()) + + # submit50 -v + # submit50 --verbose + if args["verbose"]: + run.verbose = True + + # submit50 slug + submit("submit50", args["slug"]) + + # kthxbai + sys.exit(0) + + +def authenticate(org): + """Authenticate user.""" + + # cache credentials in ~/.git-credential-cache/:org + cache = os.path.expanduser("~/.git-credential-cache") + try: + os.mkdir(cache, 0o700) + except BaseException: + pass + authenticate.SOCKET = os.path.join(cache, "submit50") + + spawn = pexpect.spawn if sys.version_info < (3, 0) else pexpect.spawnu + child = spawn( + "git -c credential.helper='cache --socket {}' credential fill".format(authenticate.SOCKET)) + child.sendline("") + if child.expect(["Username:", "Password:", pexpect.EOF]) == 2: + clear_credentials() + username, password = re.search( + "username=([^\r]+)\r\npassword=([^\r]+)", child.before, re.MULTILINE).groups() + else: + try: + username = run("git config --global credential.https://github.com/submit50.username", quiet=True) + except Error: + username = None + password = None + child.close() + + def rlinput(prompt, prefill=""): + """ + Input function that uses a prefill value and advanced line editing. + + https://stackoverflow.com/a/2533142 + """ + readline.set_startup_hook(lambda: readline.insert_text(prefill)) + try: + return input(prompt) + finally: + readline.set_startup_hook() + + # prompt for credentials + progress(False) # because not using cprint herein + if not password: + + # prompt for username, prefilling if possible + progress(False) + try: + username = rlinput(_("GitHub username: "), username).strip() + except EOFError: + print() + + # prompt for password + print(_("GitHub password: "), end="") + sys.stdout.flush() + password = str() + while True: + ch = getch() + if ch in ["\n", "\r"]: # Enter + print() + break + elif ch == "\177": # DEL + if len(password) > 0: + password = password[:-1] + print("\b \b", end="") + sys.stdout.flush() + elif ch == "\3": # ctrl-c + print("^C", end="") + os.kill(os.getpid(), signal.SIGINT) + elif ch == "\4": # ctrl-d + print() + break + else: + password += ch + print("*", end="") + sys.stdout.flush() + + # authenticate user + email = "{}@users.noreply.github.com".format(username) + res = requests.get("https://api.github.com/user", + auth=(username, password)) + + # check for 2-factor authentication + # http://github3.readthedocs.io/en/develop/examples/oauth.html?highlight=token + if "X-GitHub-OTP" in res.headers: + password = two_factor(org, username, password) + res = requests.get("https://api.github.com/user", + auth=(username, password)) + + # check if incorrect password + if res.status_code == 401: + if run.verbose: + cprint(str(res.headers), "yellow") + cprint(res.text, "yellow") + raise Error(_("Invalid username and/or password.")) + + # check for other error + elif res.status_code != 200: + if run.verbose: + cprint(str(res.headers), "yellow") + cprint(res.text, "yellow") + raise Error(_("Could not authenticate user.")) + + # canonicalize (capitalization of) username, + # especially if user logged in via email address + username = res.json()["login"] + + # cache credentials for 1 week + timeout = int(datetime.timedelta(weeks=1).total_seconds()) + run("git -c credential.helper='cache --socket {} --timeout {}' " + "-c credentialcache.ignoresighup=true credential approve".format(authenticate.SOCKET, timeout), + lines=["username={}".format(username), "password={}".format(password), "", ""], + quiet=True) + + # return credentials + return (username, password, email) + + +authenticate.SOCKET = None + + +def clear_credentials(): + """Clear git credential cache """ + run("git credential-cache --socket {} exit".format(authenticate.SOCKET)) + if platform.system() == "Darwin": + try: + run("git credential-osxkeychain erase", lines=["host=github.com", "protocol=https", ""]) + except Error: + pass + + +def cprint(text="", color=None, on_color=None, attrs=None, **kwargs): + """Colorizes text (and wraps to terminal's width).""" + + # update progress + progress(False) + + # assume 80 in case not running in a terminal + columns, lines = get_terminal_size() + if columns == 0: + columns = 80 # because get_terminal_size's default fallback doesn't work in pipes + + # print text + termcolor.cprint(textwrap.fill(text, columns, drop_whitespace=False), + color=color, on_color=on_color, attrs=attrs, **kwargs) + + +def excepthook(type, value, tb): + """Report an exception.""" + excepthook.ignore = False + progress(False) + teardown() + if type is Error and str(value): + cprint(str(value), "yellow") + elif type is requests.exceptions.ConnectionError: + cprint(_("Could not connect to GitHub."), "yellow") + else: + if run.verbose: + traceback.print_exception(type, value, tb) + cprint(_("Sorry, something's wrong! Let sysadmins@cs50.harvard.edu know!"), "yellow") + if authenticate.SOCKET: # not set when using SSH + try: + clear_credentials() + except Exception: + pass + cprint(_("Submission cancelled."), "red") + + +sys.excepthook = excepthook + + +def handler(number, frame): + """Handle SIGINT.""" + os.system("stty sane 2> {}".format(os.devnull)) # in case signalled from input_with_prefill + if progress.progressing: + progress(False) + else: + cprint() + try: + clear_credentials() + except Exception: + pass + teardown() + cprint(_("{} cancelled.".format(handler.type.capitalize())), "red") + os._exit(0) + +handler.type = "submission" + +def run(command, cwd=None, env=None, lines=[], password=None, quiet=False, timeout=None): + """Run a command.""" + + # echo command + if run.verbose: + cprint(command, attrs=["bold"]) + + # include GIT_DIR and GIT_WORK_TREE in env + if not env: + env = { + "GIT_DIR": run.GIT_DIR, + "GIT_WORK_TREE": run.GIT_WORK_TREE, + "HOME": os.path.expanduser("~") + } + if os.getenv("SSH_AGENT_PID"): + env["SSH_AGENT_PID"] = os.getenv("SSH_AGENT_PID") + if os.getenv("SSH_AUTH_SOCK"): + env["SSH_AUTH_SOCK"] = os.getenv("SSH_AUTH_SOCK") + + # spawn command + if sys.version_info < (3, 0): + child = pexpect.spawn(command, cwd=cwd, env=env, ignore_sighup=True, timeout=timeout) + else: + child = pexpect.spawnu( + command, + cwd=cwd, + encoding="utf-8", + env=env, + ignore_sighup=True, + timeout=timeout) + + # send output of command to stdout only if run with --verbose (and not quieted by caller) + if run.verbose and not quiet: + child.logfile_read = sys.stdout + + # wait for prompt, send password + if password: + i = child.expect(["Password for '.*': ", pexpect.EOF]) + if i == 0: + child.sendline(password) + + # send lines of input + for line in lines: + child.sendline(line) + + # read output, check status + command_output = child.read().strip() + child.close() + if child.signalstatus is None and child.exitstatus != 0: + raise Error() + return command_output + + +run.GIT_DIR = tempfile.mkdtemp() +run.GIT_WORK_TREE = os.getcwd() +run.verbose = False + + +def progress(message=""): + """Display a progress bar as dots.""" + + # don't show in verbose mode + if run.verbose: + if message: + print(message + "...") + return + + # stop progressing if already progressing + if progress.progressing: + progress.progressing = False + progress.thread.join() + sys.stdout.write("\n") + sys.stdout.flush() + + # display dots if message passed + if message: + def progress_helper(): + sys.stdout.write(message + "...") + sys.stdout.flush() + while progress.progressing: + sys.stdout.write(".") + sys.stdout.flush() + time.sleep(0.5) + progress.progressing = True + progress.thread = Thread(target=progress_helper) + progress.thread.start() + + +progress.progressing = False + + +def submit(org, branch): + """Submit work.""" + + # check announcements + res = requests.get("https://cs50.me/status/submit50") + if res.status_code == 200 and res.text.strip(): + raise Error(res.text.strip()) + + # require git 2.7+, so that credential-cache--daemon ignores SIGHUP + # https://github.com/git/git/blob/v2.7.0/credential-cache--daemon.c + if not which("git"): + raise Error(_("You don't have git. Install git, then re-run {}!".format(org))) + version = subprocess.check_output(["git", "--version"]).decode("utf-8") + matches = re.search(r"^git version (\d+\.\d+\.\d+).*$", version) + if not matches or StrictVersion(matches.group(1)) < StrictVersion("2.7.0"): + raise Error( + _("You have an old version of git. Install version 2.7 or later, then re-run {}!".format(org))) + + # update progress + progress("Connecting") + + # compute timestamp + global timestamp + headers = requests.get("https://api.github.com/").headers + timestamp = datetime.datetime.strptime(headers["Date"], "%a, %d %b %Y %H:%M:%S %Z") + timestamp = timestamp.strftime("%Y%m%dT%H%M%SZ") + + # check version + res = requests.get("https://cs50.me/versions/submit50") + if res.status_code != 200: + raise Error(_("You have an unknown version of submit50. " + "Email sysadmins@cs50.harvard.edu!")) + version_required = res.text.strip() + if parse_version(version_required) > parse_version(get_distribution("submit50").version): + raise Error(_("You have an old version of submit50. " + "Run update50, then re-run {}!".format(org))) + + # separate branch into slug and repo + check_repo = "@cs50/checks" + branch = branch if not branch.endswith(check_repo) else branch[:-len(check_repo)] + try: + slug, src = branch.split("@") + except ValueError: + slug, src = branch, "cs50/checks" + + # ensure slug exists + file, submit.EXCLUDE = tempfile.mkstemp() + url = "https://github.com/{}/raw/master/{}/submit50/exclude".format(src, slug) + try: + urllib.request.urlretrieve(url, filename=submit.EXCLUDE) + lines = open(submit.EXCLUDE) + except Exception as e: + if run.verbose: + cprint(str(e)) + e = Error(_("Invalid slug. Did you mean to submit something else?")) + e.__cause__ = None + raise e + if slug.startswith("/") and slug.endswith("/"): + raise Error(_("Invalid slug. Did you mean {}, without the leading and trailing slashes?".format(slug.strip("/")))) + elif slug.startswith("/"): + raise Error(_("Invalid slug. Did you mean {}, without the leading slash?".format(slug.strip("/")))) + elif slug.endswith("/"): + raise Error(_("Invalid slug. Did you mean {}, without the trailing slash?".format(slug.strip("/")))) + + # check for missing files + missing = [] + for line in lines: + matches = re.match(r"^\s*#\s*([^\s]+)\s*$", line) + if matches: + pattern = matches.group(1) + if pattern[:-1] == "/": + if not os.path.isdir(pattern): + missing.append(pattern) + elif not os.path.isfile(pattern): + missing.append(pattern) + if missing: + cprint(_("You seem to be missing these files:")) + for pattern in missing: + cprint(" {}".format(pattern)) + raise Error(_("Ensure you have the required files before submitting.")) + + # update progress + progress(_("Authenticating")) + + # authenticate user via SSH + try: + + # require ssh + assert which("ssh") + + # require GitHub username in ~/.gitconfig + username, password = run( + "git config --global credential.https://github.com/submit50.username", quiet=True), None + email = "{}@users.noreply.github.com".format(username) + repo = "git@github.com:{}/{}.git".format(org, username) + progress(False) + + # require ssh-agent + child = pexpect.spawn("ssh git@github.com") + i = child.expect(["Enter passphrase for key", "Are you sure you want to continue connecting", pexpect.EOF]) + child.close() + assert i == 2 + + # authenticate user via HTTPS + except BaseException: + username, password, email = authenticate(org) + repo = "https://{}@github.com/{}/{}".format(username, org, username) + + # update progress + progress(_("Preparing")) + + # clone repository + try: + run("git clone --bare {} {}".format(shlex.quote(repo), + shlex.quote(run.GIT_DIR)), password=password) + except BaseException: + if password: + e = Error(_("Looks like {} isn't enabled for your account yet. " + "Go to https://cs50.me/authorize and make sure you accept any pending invitations!".format(org, org))) + else: + e = Error(_("Looks like you have the wrong username in ~/.gitconfig or {} isn't yet enabled for your account. " + "Double-check ~/.gitconfig and then log into https://cs50.me/ in a browser, " + "click \"Authorize application\" if prompted, and re-run {} here.".format(org, org))) + e.__cause__ = None + raise e + + # check out .gitattributes, if any, temporarily shadowing student's, if any + if os.path.isfile(".gitattributes"): + submit.ATTRIBUTES = ".gitattributes.{}".format(round(time.time())) + os.rename(".gitattributes", submit.ATTRIBUTES) + try: + run("git checkout --force {} .gitattributes".format(branch)) + except Exception: + pass + + # set options + tag = "{}@{}".format(branch, timestamp) + run("git config user.email {}".format(shlex.quote(email))) + run("git config user.name {}".format(shlex.quote(username))) + run("git symbolic-ref HEAD refs/heads/{}".format(shlex.quote(branch))) + + # patterns of file names to exclude + run("git config core.excludesFile {}".format(shlex.quote(submit.EXCLUDE))) + + # blocklist for git-lfs + # https://github.com/git-lfs/git-lfs/blob/master/commands/command_track.go + with open("{}/info/exclude".format(run.GIT_DIR), "w") as file: + file.write(".git*\n") + file.write(".lfs*\n") + + # adds, modifies, and removes index entries to match the working tree + run("git add --all") + + # get file lists + files = run("git ls-files").splitlines() + others = run("git ls-files --exclude-from={}/info/exclude --other".format(run.GIT_DIR)).splitlines() + + # unescape any octal codes in lists + # https://stackoverflow.com/a/46650050/5156190 + def unescape(s): + if s.startswith('"') and s.endswith('"'): + return ( + s.replace('"', '') + .encode("latin1") + .decode("unicode-escape") + .encode("latin1") + .decode("utf8") + ) + return s + files = [unescape(file) for file in files] + others = [unescape(other) for other in others] + + # hide .gitattributes, if any, from output + if ".gitattributes" in files: + files.remove(".gitattributes") + + # check for large files > 100 MB (and huge files > 2 GB) + # https://help.github.com/articles/conditions-for-large-files/ + # https://help.github.com/articles/about-git-large-file-storage/ + larges, huges = [], [] + for file in files: + size = os.path.getsize(file) + if size > (100 * 1024 * 1024): + larges.append(file) + elif size > (2 * 1024 * 1024 * 1024): + huges.append(file) + if len(huges) > 0: + raise Error(_("These files are too large to be submitted:\n{}\n" + "Remove these files from your directory " + "and then re-run {}!").format("\n".join(huges), org)) + elif len(larges) > 0: + if not which("git-lfs"): + raise Error(_("These files are too large to be submitted:\n{}\n" + "Install git-lfs (or remove these files from your directory) " + "and then re-run {}!").format("\n".join(larges), org)) + run("git lfs install --local") + run("git config credential.helper cache") # for pre-push hook + for large in larges: + run("git rm --cached {}".format(large)) + run("git lfs track {}".format(large)) + run("git add {}".format(large)) + run("git add --force .gitattributes") + + # files that will be submitted + if len(files) == 0: + raise Error(_("No files in this directory are expected for submission.")) + + # prompts for submit50 + if org == "submit50": + if files: + cprint(_("Files that will be submitted:"), "green") + for file in files: + cprint("./{}".format(file), "green") + + # files that won't be submitted + if others: + cprint(_("Files that won't be submitted:"), "yellow") + for other in others: + cprint("./{}".format(other), "yellow") + + # prompt for honesty + readline.clear_history() + try: + answer = input(_("Keeping in mind the course's policy on academic honesty, " + "are you sure you want to submit these files (yes/no)? ")) + except EOFError: + answer = None + print() + if not answer or not re.match("^\s*(?:y|yes)\s*$", answer, re.I): + raise Error(_("No files were submitted.")) + + # update progress + if org == "submit50": + progress(_("Submitting")) + else: + progress(_("Uploading")) + + # push branch + run("git commit --allow-empty --message='{}'".format(timestamp)) + commit_hash = run("git rev-parse HEAD") + run("git push origin 'refs/heads/{}'".format(branch), password=password) + + # successful submission + if org == "submit50": + cprint(_("Submitted {}! See https://cs50.me/submissions.").format(branch), + "green") + progress(False) + return username, commit_hash + + +submit.ATTRIBUTES = None +submit.EXCLUDE = None + + +def teardown(): + """Delete temporary directory and temporary file, restore any attributes.""" + if os.path.isfile(".gitattributes"): + try: + os.remove(".gitattributes") + except Exception: + pass + if submit.ATTRIBUTES: + try: + os.rename(submit.ATTRIBUTES, ".gitattributes") + except Exception: + pass + shutil.rmtree(run.GIT_DIR, ignore_errors=True) + if submit.EXCLUDE: + try: + os.remove(submit.EXCLUDE) + except Exception: + pass + + +def two_factor(org, username, password): + """Get one-time authentication code.""" + # send authentication request + requests.post("https://api.github.com/authorizations", + auth=(username, password)) + while True: + cprint("Authentication code:", end=" ") + sys.stdout.flush() + code = input() + if code: + break + data = json.dumps({"scopes": ["repo", "user"], "note": "{} {}".format(org, timestamp)}) + res = requests.post("https://api.github.com/authorizations", + auth=(username, password), + data=data, + headers={"X-GitHub-OTP": str(code)}) + if res.status_code == 201 and "token" in res.json(): + return res.json()["token"] + else: + raise Error(_("Could not complete two-factor authentication.")) -setup( - author="CS50", - author_email="sysadmins@cs50.harvard.edu", - classifiers=[ - "Intended Audience :: Education", - "Programming Language :: Python :: 3", - "Topic :: Education", - "Topic :: Utilities" - ], - description="This is submit50, with which you can submit solutions to \ -problems for CS50.", - install_requires=[ - "backports.shutil_get_terminal_size", "backports.shutil_which", - "pexpect>=4.0", "requests", "six", "termcolor" - ], - keywords=["submit", "submit50"], - name="submit50", - py_modules=["submit50"], - cmdclass={ - "develop": CustomDevelop, - "install": CustomInstall - }, - entry_points={ - "console_scripts": ["submit50=submit50:main"] - }, - data_files=create_mo_files(), - url="https://github.com/cs50/submit50", - version="2.4.7" -) +if __name__ == "__main__": + main()