lcc_data.py

from collections import defaultdict as dd

masked_tags = ['###MASKED-AUTHOR###',
               '###MASKED-GROUP###',
               '###MASKED-LECTURER###']

exclude_elems_containing = [
    "http://bit.ly/1PMaL42",
    "University’s honour code and pledge",
    "consent for our assignment to be used",
    "assignment will not be marked unless"
] + masked_tags

exclude_sents_swith = [
    "name:",
    "name :",
    "names:",
    "name of student:",
    "name of student :",
    "student’s (official) name",
    "student’s name",
    "author:",
    "team members:",
    "group members:",
    "group member:",
    "members:",
    "group:",
    "group –",
    "group number:",
    "group no.:",
    "group no:",
    "by:",
    "done by:",
    "submitted by:",
    "prepared by:",
    "proposal submitted by:",
    "proposal by:",

    "matriculation number:",
    "matriculation number :",
    "matriculation no.",
    "matric no.",
    "matric no.:",
    "matric:",
    "matric :",
    "matric number:",
    "matric number :",


    "tutorial group number",
    "tutorial day",
    "tutorial group",
    "class code:",
    "class :",
    "class:",

    "tutor:",
    "faculty name:",
    "tutor name:",
    "tutor’s name",
    "for:",
    "submitted to:",
    "lecturer-in-charge:",

    "title:",
    "title of proposal:",
    "proposal title:",
    "assignment title",
    "assignment 2",

    "school name:",
    "school:",
    "school of",
    "division of",
    "university:",
    "faculty:",

    "date submitted:",
    "date submitted :",
    "date of submission:",
    "date of submission :",
    "data of submission :",
    "data of submission:",
    "date of sumission:",
    "date of sumission :",
    "data of sumission :",
    "data of sumission:",
    "date of submision:",
    "date of submision :",
    "data of submision :",
    "data of submision:",
    "date:",
    "due:",

    "engineering communication",
    "hw0188 engineering communication",
    "hw0188: engineering communication",
    "hw0188 (engineering communication)",
    "hw0188 - engineering communication",
    "course code:",
    "course:",
    "module code:",
    "module:",
    "code:",

    "cover page",

    "word count:",

    "source:",

    "diagrams:",
    "design:",

    "available at:",

    "plagiarism declaration form",
    "penalties will be imposed for late submissions",
    "i/we have read and understood the guidelines on academic dishonesty",
    "we have read and understood the guidelines on academic dishonesty",
    "i have read and understood the guidelines on academic dishonesty",
    "student’s signature:",
    "penalties will be imposed for late submission and plagiarism",
    "please refer to the general instructions for details",
    "note: the assignment will not be marked unless this form is completed and signed",
    "note: the assignment will not be marked unless",


    "figure a:", "figure a.", "figure a -", "fig a:", "fig a.", "fig a -",
    "figure b:", "figure b.", "figure b -", "fig b:", "fig b.", "fig b -",
    "figure c:", "figure c.", "figure c -", "fig c:", "fig c.", "fig c -",
    "figure d:", "figure d.", "figure d -", "fig d:", "fig d.", "fig d -",
    "figure e:", "figure e.", "figure e -", "fig e:", "fig e.", "fig e -",
    "figure f:", "figure f.", "figure f -", "fig f:", "fig f.", "fig f -",
    "figure 1:", "figure 1.", "figure 1 -", "fig 1:", "fig 1.", "fig 1 -",
    "figure 2:", "figure 2.", "figure 2 -", "fig 2:", "fig 2.", "fig 2 -",
    "figure 3:", "figure 3.", "figure 3 -", "fig 3:", "fig 3.", "fig 3 -",
    "figure 4:", "figure 4.", "figure 4 -", "fig 4:", "fig 4.", "fig 4 -",
    "figure 5:", "figure 5.", "figure 5 -", "fig 5:", "fig 5.", "fig 5 -",
    "figure 6:", "figure 6.", "figure 6 -", "fig 6:", "fig 6.", "fig 6 -",


    "table 1:", "table 1.", "table 1 -",
    "table 2:", "table 2.", "table 2 -",
    "table 3:", "table 3.", "table 3 -",
    "table 4:", "table 4.", "table 4 -",
    "table 5:", "table 5.", "table 5 -",
    "table 6:", "table 6.", "table 6 -",
    "table 7:", "table 7.", "table 7 -"
]


punctuation = """!"“”#$%&\'*+,./:;<=>?@\\^_`{|}~‘’"""

refs_section = ['reference', 'references', 'bibliography', 'source', 'sources',
                'works cited', 'references:', 'reference:', 'bibliography:',
                'source:', 'sources:', 'works cited:', 'references (apa):',
                'references (ieee):', 'citations (use apa)', 'citations',
                'citations:', 'referencing', 'refeences']

exclude_sents = [
    "costs/budget:",
    "cost/budget:",
    "cost:",
    "costs:",
    "the estimated cost per unit (SGD):",
    "the estimated cost per unit:",
    "basic cost:",
    "cost of implementation:",
    "conclusion:",
    "implementation:",
    "benefits:",
    "engineering communication i",
    "background:",
    "problem:",
    "solution:"
] + refs_section



wordchoice = set(["I would like to say",
                  "to wrap up",
                  "come up with",
                  "total waste of time",
                  "come into play",
                  "without further ado",
                  "tons of",
                  "fork out"])


# CHECK FOR PET PEEVES
wordcheck = dd(list)

# Wordcheck is checked at lemma level (important for verbs)
wordcheck['Informal'] = set(['hassle',
                             'tackle',
                             'stuff',
                             'stuffs',
                             'handy',
                             'air-con',
                             'info',
                             'fantastic',
                             'humongous',
                             'cash'])



wordcheck['Formal'] = set(['aforementioned'])

wordcheck['PronounStyle']= set(["i",
                                "me",
                                "mine",
                                "my",
                                "myself",
                                "you",
                                "your",
                                "yours",
                                "yourself"])

contractions = set(["ain't",
                    "aren't",
                    "can't",
                    "could've",
                    "couldn't",
                    "didn't",
                    "doesn't",
                    "don't",
                    "gonna",
                    "gotta",
                    "hadn't",
                    "hasn't",
                    "haven't",
                    "he'd",
                    "he'll",
                    "he's",
                    "how'd",
                    "how'll",
                    "how's",
                    "I'd",
                    "I'll",
                    "I'm",
                    "I've",
                    "isn't",
                    "it'd",
                    "it'll",
                    "it's",
                    "mayn't",
                    "may've",
                    "mightn't",
                    "might've",
                    "mustn't",
                    "must've",
                    "needn't",
                    "oughtn't",
                    "shan't",
                    "she'd",
                    "she'll",
                    "she's",
                    "should've",
                    "shouldn't",
                    "that'll",
                    "that're",
                    "that's",
                    "that'd",
                    "there'd",
                    "there're",
                    "there's",
                    "these're",
                    "they'd",
                    "they'll",
                    "they're",
                    "they've",
                    "this's",
                    "those're",
                    "wasn't",
                    "we'd",
                    "we'll",
                    "we're",
                    "we've",
                    "weren't",
                    "what'd",
                    "what'll",
                    "what're",
                    "what's",
                    "what've",
                    "when's",
                    "where'd",
                    "where're",
                    "where's",
                    "where've",
                    "which's",
                    "who'd",
                    "who'll",
                    "who're",
                    "who's",
                    "who've",
                    "why'd",
                    "why're",
                    "why's",
                    "won't",
                    "would've",
                    "wouldn't",
                    "y'all",
                    "you'd",
                    "you'll",
                    "you're",
                    "you've"])

new_contractions = set([])
for c in contractions:
    new_contractions.add(c.replace("'","’"))
contractions = contractions | new_contractions

wordcase_ntu = set(["Executive Committee",
                    "Student Services Center",
                    "Student Services Centre",
                    "Jurong East",
                    "NTU",
                    "Student Union",
                    "Computer Science",
                    "Google",
                    "Microsoft",
                    "Singaporeans",
                    "Singaporean",
                    "Singapore",
                    "School of Engineering",
                    "Nanyang Techonological University",
                    "MRT",
                    "EZ-Link"])

months = set(["January",
              "February",
              "March",
              "April",
               # "May", is ambigous, leaving it out
              "June",
              "July",
              "August",
              "September",
              "October",
              "November",
              "December"])

places = set(["Alexandra",
              "Aljunied",
              "Geylang",
              "Ayer Rajah",
              "Balestier",
              "Bartley",
              "Bishan",
              "Marymount",
              "Sin Ming",
              "Bukit Timah",
              "Sixth Avenue",
              "Buona Vista",
              "Holland Village",
              "One North",
              "Ghim Moh",
              "Chinatown",
              "Clarke Quay",
              "Kreta Ayer",
              "Telok Ayer",
              "Kallang",
              "Bendemeer",
              "Geylang Bahru",
              "Kallang Bahru",
              "Kallang Basin",
              "Kolam Ayer",
              "Tanjong Rhu",
              "Mountbatten",
              "Old Airport",
              "Lavender",
              "Boon Keng",
              "Kent Ridge",
              "Kim Seng",
              "Little India",
              "Farrer Park",
              "Jalan Besar",
              "MacPherson",
              "Marina Bay",
              "Esplanade",
              "Marina Bay Sands",
              "Marina Centre",
              "Marina East",
              "Marina South",
              "Mount Faber",
              "Mount Vernon",
              "Museum",
              "Newton",
              "Novena",
              "Orchard Road",
              "Dhoby Ghaut",
              "Emerald Hill",
              "Peranakan Place",
              "Tanglin",
              "Outram",
              "Pasir Panjang",
              "Paya Lebar",
              "Eunos",
              "Geylang East",
              "Potong Pasir",
              "Rochor-Kampong Glam",
              "Bencoolen",
              "Bras Basah",
              "Bugis",
              "Queenstown",
              "Dover",
              "Commonwealth",
              "Raffles Place",
              "River Valley",
              "Singapore River",
              "Southern Islands",
              "Tanjong Pagar",
              "Shenton Way",
              "Telok Blangah",
              "Bukit Chandu",
              "Bukit Purmei",
              "HarbourFront",
              "Keppel",
              "Radin Mas",
              "Mount Faber",
              "Tiong Bahru",
              "Bukit Ho Swee",
              "Bukit Merah",
              "Toa Payoh",
              "Bukit Brown",
              "Caldecott Hill",
              "Thomson",
              "Whampoa",
              "St. Michael's",
              "Bedok",
              "Bedok Reservoir",
              "Chai Chee",
              "Kaki Bukit",
              "Tanah Merah",
              "Changi",
              "Changi Bay",
              "Changi East",
              "Changi Village",
              "East Coast",
              "Joo Chiat",
              "Katong",
              "Kembangan",
              "Pasir Ris",
              "Elias",
              "Lorong Halus",
              "Loyang",
              "Marine Parade",
              "Siglap",
              "Tampines",
              "Simei",
              "Ubi",
              "Central Catchment Nature Reserve",
              "Kranji",
              "Lentor",
              "Lim Chu Kang",
              "Neo Tiew",
              "Sungei Gedong",
              "Mandai",
              "Sembawang",
              "Canberra",
              "Senoko",
              "Simpang",
              "Sungei Kadut",
              "Woodlands",
              "Admiralty",
              "Innova",
              "Marsiling",
              "Woodgrove",
              "Yishun",
              "Chong Pang",
              "Ang Mo Kio",
              "Cheng San",
              "Chong Boon",
              "Kebun Baru",
              "Teck Ghee",
              "Yio Chu Kang",
              "Bidadari",
              "Hougang",
              "Defu",
              "Kovan",
              "Lorong Chuan",
              "North-Eastern Islands",
              "Punggol",
              "Punggol Point",
              "Punggol New Town",
              "Seletar",
              "Sengkang",
              "Serangoon",
              "Serangoon Gardens",
              "Serangoon North",
              "Boon Lay",
              "Tukang",
              "Liu Fang",
              "Samulun",
              "Shipyard",
              "Bukit Batok",
              "Bukit Gombak",
              "Hillview",
              "Guilin",
              "West",
              "East",
              "Bukit Panjang",
              "Choa Chu Kang",
              "Yew Tee",
              "Clementi",
              "Toh Tuck",
              "West Coast",
              "Jurong East",
              "Toh Guan",
              "International Business Park",
              "Teban Gardens",
              "Pandan Gardens",
              "Penjuru",
              "Yuhua",
              "Jurong Regional Centre",
              "Lake",
              "River",
              "Port",
              "Jurong West",
              "Hong Kah",
              "Taman Jurong",
              "Boon Lay Place",
              "Chin Bee",
              "Yunnan",
              "Kian Teck",
              "Safti",
              "Wenya",
              "Lim Chu Kang",
              "Pioneer",
              "Joo Koon",
              "Gul Circle",
              "Pioneer Sector",
              "Tengah",
              "Tuas",
              "Wrexham",
              "Promenade",
              "Pioneer",
              "Soon Lee",
              "Tuas South",
              "Western Islands Planning Area",
              "Western Water Catchment",
              "Murai",
              "Sarimbun"])


wordcase = wordcase_ntu | months | places