From 3f385d0cdf07836e822115167ef25c185ffacd14 Mon Sep 17 00:00:00 2001
From: Crinibus <57172157+Crinibus@users.noreply.github.com>
Date: Tue, 13 Oct 2020 23:44:28 +0200
Subject: [PATCH 1/5] Add contributing section
---
README.md | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/README.md b/README.md
index 6a7758b0..7dee56d5 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
# Table of contents
+- [Contributing](#contributing)
- [First setup](#first-setup)
- [Tech scraper](#tech-scraper)
- [Scrape products](#scrape-products)
@@ -10,6 +11,9 @@
+## Contributing
+Feel free to fork the project and create a pull request with new features or refactoring of the code.
+
## First setup
First make sure you have the modules, run this in the terminal:
From 7bbb97e33870252d00dd42f88e0ae2024c76bb77 Mon Sep 17 00:00:00 2001
From: Crinibus <57172157+Crinibus@users.noreply.github.com>
Date: Wed, 14 Oct 2020 11:41:38 +0200
Subject: [PATCH 2/5] Change function find_domain
- replace all the if-statements with a dictionary
---
tech_scraper/add_product.py | 44 +++++++++++++++----------------------
1 file changed, 18 insertions(+), 26 deletions(-)
diff --git a/tech_scraper/add_product.py b/tech_scraper/add_product.py
index b8739071..6fdc2906 100644
--- a/tech_scraper/add_product.py
+++ b/tech_scraper/add_product.py
@@ -162,32 +162,24 @@ def save_json(category, product_name):
def find_domain(domain):
"""Return the domain name of the url. Used to determine which class to call in scrape_link.py"""
- if domain == domains['komplett']:
- return 'Komplett'
- elif domain == domains['proshop']:
- return 'Proshop'
- elif domain == domains['computersalg']:
- return 'Computersalg'
- elif domain == domains['elgiganten']:
- return 'Elgiganten'
- elif domain == domains['avxperten']:
- return 'AvXperten'
- elif domain == domains['av-cables']:
- return 'AvCables'
- elif domain == domains['amazon']:
- return 'Amazon'
- elif domain == domains['ebay']:
- return 'eBay'
- elif domain == domains['power']:
- return 'Power'
- elif domain == domains['expert']:
- return 'Expert'
- elif domain == domains['mm-vision']:
- return 'MMVision'
- elif domain == domains['coolshop']:
- return 'Coolshop'
- elif domain == domains['sharkgaming']:
- return 'Sharkgaming'
+
+ class_domains = {
+ "www.komplett.dk": "Komplett",
+ "www.proshop.dk": "Proshop",
+ "www.computersalg.dk": "Computersalg",
+ "www.elgiganten.dk": "Elgiganten",
+ "www.avxperten.dk": "AvXperten",
+ "www.av-cables.dk": "AvCables",
+ "www.amazon.com": "Amazon",
+ "www.ebay.com": "eBay",
+ "www.power.dk": "Power",
+ "www.expert.dk": "Expert",
+ "www.mm-vision.dk": "MMVision",
+ "www.coolshop.dk": "Coolshop",
+ "sharkgaming.dk": "Sharkgaming",
+ }
+
+ return class_domains[domain]
def add_to_scraper(category, link, url_domain):
From d0fd8f9e0f53505533a74134ccd12a0dd0407756 Mon Sep 17 00:00:00 2001
From: Crinibus <57172157+Crinibus@users.noreply.github.com>
Date: Wed, 14 Oct 2020 11:48:11 +0200
Subject: [PATCH 3/5] Update contributing section
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 7dee56d5..4867194a 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
## Contributing
-Feel free to fork the project and create a pull request with new features or refactoring of the code.
+Feel free to fork the project and create a pull request with new features or refactoring of the code. Also feel free to make a issue with problems or suggestions to a new feature.
## First setup
First make sure you have the modules, run this in the terminal:
From 1e8e515406491bd6942dc6837faa0be8041fdcc8 Mon Sep 17 00:00:00 2001
From: Crinibus <57172157+Crinibus@users.noreply.github.com>
Date: Wed, 14 Oct 2020 11:49:47 +0200
Subject: [PATCH 4/5] Add line break between contributing and first step
sections
---
README.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/README.md b/README.md
index 4867194a..313c6e15 100644
--- a/README.md
+++ b/README.md
@@ -14,6 +14,8 @@
## Contributing
Feel free to fork the project and create a pull request with new features or refactoring of the code. Also feel free to make a issue with problems or suggestions to a new feature.
+
+
## First setup
First make sure you have the modules, run this in the terminal:
From ff9e6224bc7862002a98414a858a1cf770eabb89 Mon Sep 17 00:00:00 2001
From: Crinibus <57172157+Crinibus@users.noreply.github.com>
Date: Wed, 14 Oct 2020 17:17:48 +0200
Subject: [PATCH 5/5] Make some of the code over multiple lines
---
tech_scraper/scraper.py | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/tech_scraper/scraper.py b/tech_scraper/scraper.py
index 72f13bad..fd46b486 100644
--- a/tech_scraper/scraper.py
+++ b/tech_scraper/scraper.py
@@ -42,12 +42,18 @@ def __init__(self, category, URL):
try:
self.get_response()
except Exception as err:
- self.logger.error(f'Failed in method "{self.__class__.__name__}.get_response()": {err}', exc_info=True)
+ self.logger.error(
+ f'Failed in method "{self.__class__.__name__}.get_response()": {err}',
+ exc_info=True
+ )
try:
self.get_info()
except Exception as err:
- self.logger.error(f'Failed in method "{self.__class__.__name__}.get_info()": {err}', exc_info=True)
+ self.logger.error(
+ f'Failed in method "{self.__class__.__name__}.get_info()": {err}',
+ exc_info=True
+ )
self.name = change_æøå(change_name(self.name))
self.date = str(datetime.today().strftime('%Y-%m-%d'))
@@ -58,7 +64,10 @@ def __init__(self, category, URL):
try:
self.save_record()
except Exception as err:
- self.logger.error(f'Failed in method "{self.__class__.__name__}.save_record()": {err}', exc_info=True)
+ self.logger.error(
+ f'Failed in method "{self.__class__.__name__}.save_record()": {err}',
+ exc_info=True
+ )
def get_info(self): # gets overwritten
"""Get name and price of product."""
@@ -225,7 +234,10 @@ def change_name(name):
def change_æøå(name):
- """Change the letters æ, ø and å to international letters to avoid unicode and return the new name."""
+ """
+ Change the letters æ, ø and å to international letters to avoid unicode and
+ return the new name.
+ """
replace_letters = {
"æ": "ae",
"ø": "oe",