Merge branch 'develop' into dependabot/cargo/develop/reqwest_cookie_s…

…tore-0.8
ThalosES · Jul 28, 2024 · ce42433 · ce42433
2 parents fe16a41 + 3d9f012
commit ce42433
Show file tree

Hide file tree

Showing 65 changed files with 105,309 additions and 13 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,3 +3,5 @@ Cargo.lock
 .vscode/
 /**/results/
 docker/query.sh
+
+src/tests/output
diff --git a/.gitmodules b/.gitmodules
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,15 @@
 # Changelog
 
+# 0.9.0 (2024-07-28) [#97](https://github.com/TuTarea/vinted-rs/pull/97/)
+
+## Fixed
+
+- Updated API fields for `advancedItem`
+
+## Improved
+
+- Moved the [scrapping scripts](https://github.com/ThalosES/vinted-db-feeder) in `/scrapping` folder and removed the old repo
+
 # 0.8.7 (2024-04-3) [#82](https://github.com/TuTarea/vinted-rs/pull/82/)
 
 ## Improved

diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "vinted-rs"
-version = "0.8.7"
+version = "0.9.0"
 edition = "2021"
 repository = "https://github.com/TuTarea/vinted-rs"
 authors = [
@@ -39,6 +39,7 @@ serde_json = { version = "1.0.91" }
 log = "0.4.20"
 lazy_static = "1.4.0"
 [dev-dependencies]
+env_logger = "0.11.5"
 redis-macros = { version = "0.3.0" }
 redis = { version = "0.24.0" }
 

diff --git a/README.md b/README.md
@@ -24,7 +24,7 @@ Via `cargo` you can add the library to your project's `Cargo.toml`
 
 ```toml
 [dependencies]
-vinted-rs = "0.8.7"
+vinted-rs = "0.9.0"
 ```
 
 ## DB setup

diff --git a/scrapping/vinted-db-feeder/.gitignore b/scrapping/vinted-db-feeder/.gitignore
@@ -0,0 +1,5 @@
+data/*
+!data/raw
+
+pyvenv
+*/__pycache__
diff --git a/scrapping/vinted-db-feeder/README.md b/scrapping/vinted-db-feeder/README.md
@@ -0,0 +1,76 @@
+# Vinted data scrapper
+
+A series of data scrappers in Python that extract static information from the Vinted source code, being the perfect complement for a Vinted API wrapper.
+
+## Install & Run
+
+### Requirements
+
+- Pip
+- Python
+
+1. Create a virtual environment
+
+   ```bash
+   python3 -m venv pyvenv/
+   source venv/bin/activate
+   ```
+
+2. Run `requirements.txt`
+
+   ```bash
+    pip3 install -r requirements.txt
+   ```
+
+3. Run `main.py`
+
+## Extracted data categories
+
+| Element             | Fields Returned                                     |
+| ------------------- | --------------------------------------------------- |
+| Brands              | Names, Ids                                          |
+| Materials           | Id, Name                                            |
+| Colors              | Id, Color, Hex Code                                 |
+| Sizes               | Id, Title, Size_Type, Category_id                   |
+| Categories          | Id, Title, Code, Parent Id, URL, URL EN, Item Count |
+| Categories Children | Category Id, Child Id                               |
+| Countries           | Id, French_name, local_name, ISO_code, flag_emoji   |
+
+### Materials and sizes
+
+- Available languages: 🇪🇸 🇫🇷 🇺🇸
+  - [More languages can be added if html file included in `data/raw/materials` or `data/raw/sizes`]
+
+### Categories - Catalogs
+
+- **Debug mode:** Builds the full decision tree
+
+- **Exec mode:** Returns 2 CSVs:
+
+  - `categories.csv`: Table of all the available categories and their attributes
+
+  - `categories_children.csv`: Dictionary that models the Category->Children list relationship
+
+## Performance
+
+- Without brands search:
+
+```bash
+real    0m1,941s
+user    0m1,225s
+sys     0m0,037s
+```
+
+- Brands validation process:
+
+```bash
+real    14m14,211s
+user    0m19,753s
+sys     0m1,229s
+```
+
+## Authors
+
+[Álvaro Cabo](https://github.com/alvarocabo)
+
+[Pepe Márquez](https://github.com/pxp9)
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/A_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/A_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/B_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/B_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/C_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/C_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/D_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/D_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/E_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/E_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/F_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/F_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/G_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/G_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/H_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/H_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/I_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/I_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/J_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/J_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/K_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/K_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/L_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/L_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/M_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/M_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/N_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/N_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/O_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/O_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/P_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/P_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/R_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/R_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/S_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/S_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/T_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/T_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/U_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/U_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/V_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/V_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/W_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/W_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/Y_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/Y_elements.html
diff --git a/scrapping/vinted-db-feeder/data/raw/brands/Z_elements.html b/scrapping/vinted-db-feeder/data/raw/brands/Z_elements.html