-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
c767fdb
commit 6ce9b04
Showing
5 changed files
with
241 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
*.box | ||
*.pyc | ||
.*.sw* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
|
||
import urllib2 | ||
from bs4 import BeautifulSoup # import required modules for web scraping | ||
|
||
names=["GPS", "NFLX", "GRPN", "INTC", "AAPL", "MOLG", "WAIR", "OTIV", "AMDA", "ARUN", "ICLDW", "AAPL", "GOOG", "BABA"] | ||
|
||
def calc(names): # method declaration | ||
for i in names: # loop start to use the yahoo finace link on each stock name | ||
url = 'http://finance.yahoo.com/q?s='+i # the url to visit and scrape for news articles | ||
data = urllib2.urlopen(url) # getting the data from the page | ||
soup = BeautifulSoup(data) # letting beautiful soup 4 work its magic on the data | ||
|
||
divs = soup.find('div',attrs={'id':'yfi_headlines'}) # looking for the particular tag | ||
div = divs.find('div',attrs={'class':'bd'}) # looking for a tag under a tag | ||
ul = div.find('ul') | ||
lis = ul.findAll('li') | ||
print i # print the stock you are now dealing with | ||
m=0.0 # score saver for results on using sentiment analysis | ||
for li in lis: # looping through tags that match the required web scraping criteria | ||
headlines = li.find('a').get('href') # getting the url in the tags, linking to actual articles | ||
print headlines | ||
ur='http://access.alchemyapi.com/calls/url/URLGetTextSentiment?apikey=9b61009a54069badce0cc7ed6bc3f229b07d150a&url='+headlines | ||
dat=urllib2.urlopen(ur) # beautiful soup again but on the sentiment analysis results | ||
sou=BeautifulSoup(dat) # beautiful soup again but on the sentiment analysis results | ||
if sou.find('score')!=None: | ||
m+=float(sou.find('score').string) | ||
return m | ||
|
||
print calc(names) # print the output |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
|
||
|
||
|
||
|
||
# Vagrant | ||
|
||
Vagrant is a virtual machine manager that can be shared across platforms. | ||
|
||
Download and install vagrant from [Vagrant's website](https://www.vagrantup.com/downloads.html). | ||
|
||
The vagrant box can be found under the "Releases" tab of this repository. Download it and follow the Usage instructions | ||
|
||
### Usage | ||
To add the vagrant box to local vagrant: | ||
```bash | ||
vagrant box add --name adsa/python [path to vagrant box] | ||
``` | ||
Now the box will be available locally. You can use it by copying the Vagrantfile in this repository | ||
or by running | ||
```bash | ||
vagrant init | ||
``` | ||
and changing the generated Vagrantfile line | ||
```ruby | ||
config.vm.box = "base" | ||
``` | ||
to | ||
```ruby | ||
config.vm.box = "adsa/python" | ||
``` | ||
|
||
|
||
Now, in the folder that contains the Vagrantfile, run | ||
```bash | ||
vagrant up | ||
vagrant ssh | ||
``` | ||
|
||
This will log you in to the virtual machine and give you console access. Files in the folder containing | ||
the Vagrantfile will be shared to the ```/vagrant``` folder in the virtual machine, so you can edit | ||
files in your normal OS with your text editor of choice (Sublime Text, Atom, etc.). |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
# -*- mode: ruby -*- | ||
# vi: set ft=ruby : | ||
|
||
# All Vagrant configuration is done below. The "2" in Vagrant.configure | ||
# configures the configuration version (we support older styles for | ||
# backwards compatibility). Please don't change it unless you know what | ||
# you're doing. | ||
Vagrant.configure(2) do |config| | ||
# The most common configuration options are documented and commented below. | ||
# For a complete reference, please see the online documentation at | ||
# https://docs.vagrantup.com. | ||
|
||
# Every Vagrant development environment requires a box. You can search for | ||
# boxes at https://atlas.hashicorp.com/search. | ||
config.vm.box = "adsa/python" | ||
|
||
# Disable automatic box update checking. If you disable this, then | ||
# boxes will only be checked for updates when the user runs | ||
# `vagrant box outdated`. This is not recommended. | ||
# config.vm.box_check_update = false | ||
|
||
# Create a forwarded port mapping which allows access to a specific port | ||
# within the machine from a port on the host machine. In the example below, | ||
# accessing "localhost:8080" will access port 80 on the guest machine. | ||
# config.vm.network "forwarded_port", guest: 80, host: 8080 | ||
|
||
# Create a private network, which allows host-only access to the machine | ||
# using a specific IP. | ||
# config.vm.network "private_network", ip: "192.168.33.10" | ||
|
||
# Create a public network, which generally matched to bridged network. | ||
# Bridged networks make the machine appear as another physical device on | ||
# your network. | ||
# config.vm.network "public_network" | ||
|
||
# Share an additional folder to the guest VM. The first argument is | ||
# the path on the host to the actual folder. The second argument is | ||
# the path on the guest to mount the folder. And the optional third | ||
# argument is a set of non-required options. | ||
# config.vm.synced_folder "../data", "/vagrant_data" | ||
|
||
# Provider-specific configuration so you can fine-tune various | ||
# backing providers for Vagrant. These expose provider-specific options. | ||
# Example for VirtualBox: | ||
# | ||
# config.vm.provider "virtualbox" do |vb| | ||
# # Display the VirtualBox GUI when booting the machine | ||
# vb.gui = true | ||
# | ||
# # Customize the amount of memory on the VM: | ||
# vb.memory = "1024" | ||
# end | ||
# | ||
# View the documentation for the provider you are using for more | ||
# information on available options. | ||
|
||
# Define a Vagrant Push strategy for pushing to Atlas. Other push strategies | ||
# such as FTP and Heroku are also available. See the documentation at | ||
# https://docs.vagrantup.com/v2/push/atlas.html for more information. | ||
# config.push.define "atlas" do |push| | ||
# push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" | ||
# end | ||
|
||
# Enable provisioning with a shell script. Additional provisioners such as | ||
# Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the | ||
# documentation for more information about their specific syntax and use. | ||
# config.vm.provision "shell", inline: <<-SHELL | ||
# sudo apt-get update | ||
# sudo apt-get upgrade | ||
# sudo apt-get install python python-dev libmysqlclient-dev | ||
# wget https://bootstrap.pypa.io/get-pip.py && python get-pip.py | ||
# sudo pip install mysql-python requests | ||
# sudo apt-get install mysql | ||
# SHELL | ||
end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
import requests | ||
import json | ||
import MySQLdb | ||
import datetime | ||
import sys | ||
|
||
# Script to call the UIUC EWS lab utilization api | ||
# DOES NOT ACCOUNT FOR TIMEZONE!!! | ||
|
||
# MYSQL datetime format | ||
time_format = '%Y-%m-%d %H:%M:%S' | ||
|
||
|
||
|
||
|
||
|
||
|
||
def fetch(link): | ||
""" | ||
Parameters: URL to page to read | ||
Returns: JSON from page | ||
""" | ||
# Use requests library to request the file | ||
r = requests.get(link) | ||
# Make sure status code is within acceptable range. See: http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html | ||
if(r.status_code >= 200 and r.status_code < 300): | ||
# Throw it as error if not in 2xx | ||
r.raise_for_status() | ||
# Convert from JSON to python dictionary | ||
data = json.loads(r.text) | ||
return data | ||
|
||
|
||
def store(json, config): | ||
""" | ||
Parameters: JSON DB Config object of for | ||
{ | ||
"host": ..., | ||
"user": ..., | ||
"passwd": ..., | ||
"db": ..., | ||
} | ||
""" | ||
# Snag the data from the "data" key | ||
data = json["data"] | ||
|
||
# Connect to the mysql database from config dictionary | ||
# Note: ** is the unwinding operator. Turns a dictionary into named parameters | ||
# Documentation for MySQLdb is here: http://mysql-python.sourceforge.net/MySQLdb.html | ||
db = MySQLdb.connect(**config) | ||
cursor = db.cursor() | ||
for lab in data: | ||
table = lab["strlabname"].replace(" ", "_") | ||
columns = ", ".join(["timestamp", "inusecount" , "machinecount"]) | ||
values = ", ".join(["'" + str(datetime.datetime.now().strftime(time_format)) + "'", str(lab["inusecount"]), str(lab["machinecount"])]) | ||
# SQL Insert statement: see http://www.w3schools.com/sql/sql_insert.asp | ||
# """ Means keep the formatting of the string | ||
sql = """ | ||
INSERT INTO %s | ||
( %s ) | ||
VALUES ( %s ) | ||
""" % (table, columns, values) | ||
# "Executes" the sql string but doesn't actually commit the result | ||
cursor.execute(sql) | ||
# Commits the change to the database | ||
db.commit() | ||
db.close() | ||
return True | ||
|
||
|
||
# Code the is equivalent to "int main() {}" | ||
if __name__=="__main__": | ||
""" | ||
Called from cron job, get and store data | ||
""" | ||
link = "https://my.engr.illinois.edu/labtrack/util_data_json.asp" | ||
if len(sys.argv) != 2: | ||
print """ | ||
Usage: python %s <path to json config file> | ||
""" % sys.argv[0] | ||
sys.exit(1) | ||
with open(sys.argv[1], "r") as conf: | ||
config = json.load(conf) | ||
|
||
try: | ||
# Use the defined functions | ||
data = fetch(link) | ||
# Store the value | ||
success = store(data, config) | ||
if not success: | ||
print "%s" % datetime.datetime.now().strftime(time_format) | ||
except Exception as e: | ||
print e |