Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add first exploration to detect vandalism #353

Merged
merged 4 commits into from
Nov 4, 2024
Merged

feat: add first exploration to detect vandalism #353

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2bb5385
initial commit
baslia Sep 23, 2024
0d60fe8
update notebook
baslia Oct 2, 2024
782463e
add option to redownload new file
baslia Oct 2, 2024
86526ce
update notebook
baslia Nov 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions genai-features/dataset/recent_changes.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
http://static.openfoodfacts.org/data/openfoodfacts_recent_changes.jsonl.gz
367 changes: 367 additions & 0 deletions genai-features/notebooks/explore_recent_changes.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,367 @@
{
"cells": [
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2024-10-02T05:33:00.083862Z",
"start_time": "2024-10-02T05:32:59.258183Z"
}
},
"source": [
"import pandas as pd\n",
"import json\n",
"import requests"
],
"outputs": [],
"execution_count": 1
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2024-10-02T05:33:00.094035Z",
"start_time": "2024-10-02T05:33:00.092103Z"
}
},
"cell_type": "code",
"source": [
"n_sample = None\n",
"re_download = False"
],
"id": "5471642089fb0a62",
"outputs": [],
"execution_count": 2
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2024-10-02T05:34:22.364615Z",
"start_time": "2024-10-02T05:33:00.106532Z"
}
},
"cell_type": "code",
"source": [
"data_url = 'http://static.openfoodfacts.org/data/openfoodfacts_recent_changes.jsonl.gz'\n",
"\n",
"if re_download:\n",
" import gzip\n",
" import shutil\n",
" # Download the data\n",
" data_path = '/Users/baslad01/data_dump'\n",
" file_path = f'{data_path}/openfoodfacts_recent_changes.jsonl.gz'\n",
" response = requests.get(data_url)\n",
" with open(file_path, 'wb') as file:\n",
" file.write(response.content)\n",
" \n",
" compressed_file_path = f'{data_path}/openfoodfacts_recent_changes.jsonl.gz'\n",
" uncompressed_file_path = f'{data_path}/openfoodfacts_recent_changes.jsonl'\n",
"\n",
" \n",
" with gzip.open(compressed_file_path, 'rb') as f_in:\n",
" with open(uncompressed_file_path, 'wb') as f_out:\n",
" shutil.copyfileobj(f_in, f_out)\n",
"\n",
"\n",
"data_path = '/Users/baslad01/data_dump'\n",
"file_path = f'{data_path}/openfoodfacts_recent_changes.jsonl'\n",
"key_words = ['vandal']\n",
"\n",
"filtered_data = []\n",
"\n",
"with open(file_path, 'r') as file:\n",
" for line in file:\n",
" try:\n",
" json_obj = json.loads(line)\n",
" if 'comment' in json_obj and any(kw.lower() in json_obj['comment'].lower() for kw in key_words):\n",
" filtered_data.append(json_obj)\n",
" except json.JSONDecodeError as e:\n",
" print(f\"Error decoding JSON: {e}\")\n",
"\n",
"df_recent_changes_filtered = pd.DataFrame(filtered_data)\n",
"df_recent_changes_filtered"
],
"id": "56b61ebb99017c3d",
"outputs": [
{
"data": {
"text/plain": [
" _id userid code \\\n",
"0 {'$oid': '5bbcdc0a4ade5fdf2732e301'} sebleouf 3596654383769 \n",
"1 {'$oid': '5bbce3b24ade5f069444b8ce'} sebleouf 8010059016480 \n",
"2 {'$oid': '5bc064534ade5fee8676c08a'} sebleouf 9789045548647 \n",
"3 {'$oid': '5bc373984ade5f613e7f885c'} sebleouf 6922572400030 \n",
"4 {'$oid': '5bc4674c4ade5fa734766af5'} sebleouf 3515450030899 \n",
"... ... ... ... \n",
"1578 {'$oid': '66d886631ec3700d69da1183'} charlesnepote 0810554026773 \n",
"1579 {'$oid': '66d8868637b2e30ab8da1180'} charlesnepote 3760144210563 \n",
"1580 {'$oid': '66d886a906b365812fda1180'} charlesnepote 3760282062437 \n",
"1581 {'$oid': '66d8871362cae1c7fcda1180'} charlesnepote 8052282080203 \n",
"1582 {'$oid': '66d887396aabbb9bc9da1181'} charlesnepote 4056489774877 \n",
"\n",
" comment countries_tags diffs t rev \n",
"0 Suppression du produit :Vandalisme [en:france] {} 1539103754 8 \n",
"1 Suppression du produit :Vandalisme [en:france] {} 1539105714 8 \n",
"2 Suppression du produit :Vandalisme [en:belgium] {} 1539335251 6 \n",
"3 Suppression du produit :Vandalisme [en:belgium] {} 1539535765 6 \n",
"4 Deleting product:Vandalisme [en:algeria] {} 1539598156 5 \n",
"... ... ... ... ... ... \n",
"1578 Deleting product:Vandalism [en:france] {} 1725466211 7 \n",
"1579 Deleting product:Vandalism [en:france] {} 1725466246 7 \n",
"1580 Deleting product:Vandalism [en:france] {} 1725466281 7 \n",
"1581 Deleting product:Vandalism [en:france] {} 1725466387 10 \n",
"1582 Deleting product:Vandalism [en:france] {} 1725466425 9 \n",
"\n",
"[1583 rows x 8 columns]"
],
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>_id</th>\n",
" <th>userid</th>\n",
" <th>code</th>\n",
" <th>comment</th>\n",
" <th>countries_tags</th>\n",
" <th>diffs</th>\n",
" <th>t</th>\n",
" <th>rev</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>{'$oid': '5bbcdc0a4ade5fdf2732e301'}</td>\n",
" <td>sebleouf</td>\n",
" <td>3596654383769</td>\n",
" <td>Suppression du produit :Vandalisme</td>\n",
" <td>[en:france]</td>\n",
" <td>{}</td>\n",
" <td>1539103754</td>\n",
" <td>8</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>{'$oid': '5bbce3b24ade5f069444b8ce'}</td>\n",
" <td>sebleouf</td>\n",
" <td>8010059016480</td>\n",
" <td>Suppression du produit :Vandalisme</td>\n",
" <td>[en:france]</td>\n",
" <td>{}</td>\n",
" <td>1539105714</td>\n",
" <td>8</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>{'$oid': '5bc064534ade5fee8676c08a'}</td>\n",
" <td>sebleouf</td>\n",
" <td>9789045548647</td>\n",
" <td>Suppression du produit :Vandalisme</td>\n",
" <td>[en:belgium]</td>\n",
" <td>{}</td>\n",
" <td>1539335251</td>\n",
" <td>6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>{'$oid': '5bc373984ade5f613e7f885c'}</td>\n",
" <td>sebleouf</td>\n",
" <td>6922572400030</td>\n",
" <td>Suppression du produit :Vandalisme</td>\n",
" <td>[en:belgium]</td>\n",
" <td>{}</td>\n",
" <td>1539535765</td>\n",
" <td>6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>{'$oid': '5bc4674c4ade5fa734766af5'}</td>\n",
" <td>sebleouf</td>\n",
" <td>3515450030899</td>\n",
" <td>Deleting product:Vandalisme</td>\n",
" <td>[en:algeria]</td>\n",
" <td>{}</td>\n",
" <td>1539598156</td>\n",
" <td>5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1578</th>\n",
" <td>{'$oid': '66d886631ec3700d69da1183'}</td>\n",
" <td>charlesnepote</td>\n",
" <td>0810554026773</td>\n",
" <td>Deleting product:Vandalism</td>\n",
" <td>[en:france]</td>\n",
" <td>{}</td>\n",
" <td>1725466211</td>\n",
" <td>7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1579</th>\n",
" <td>{'$oid': '66d8868637b2e30ab8da1180'}</td>\n",
" <td>charlesnepote</td>\n",
" <td>3760144210563</td>\n",
" <td>Deleting product:Vandalism</td>\n",
" <td>[en:france]</td>\n",
" <td>{}</td>\n",
" <td>1725466246</td>\n",
" <td>7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1580</th>\n",
" <td>{'$oid': '66d886a906b365812fda1180'}</td>\n",
" <td>charlesnepote</td>\n",
" <td>3760282062437</td>\n",
" <td>Deleting product:Vandalism</td>\n",
" <td>[en:france]</td>\n",
" <td>{}</td>\n",
" <td>1725466281</td>\n",
" <td>7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1581</th>\n",
" <td>{'$oid': '66d8871362cae1c7fcda1180'}</td>\n",
" <td>charlesnepote</td>\n",
" <td>8052282080203</td>\n",
" <td>Deleting product:Vandalism</td>\n",
" <td>[en:france]</td>\n",
" <td>{}</td>\n",
" <td>1725466387</td>\n",
" <td>10</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1582</th>\n",
" <td>{'$oid': '66d887396aabbb9bc9da1181'}</td>\n",
" <td>charlesnepote</td>\n",
" <td>4056489774877</td>\n",
" <td>Deleting product:Vandalism</td>\n",
" <td>[en:france]</td>\n",
" <td>{}</td>\n",
" <td>1725466425</td>\n",
" <td>9</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>1583 rows × 8 columns</p>\n",
"</div>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": 3
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2024-10-02T05:39:14.162753Z",
"start_time": "2024-10-02T05:39:14.011972Z"
}
},
"cell_type": "code",
"source": [
"product_id = 875444\n",
"rev_id = 3\n",
"api_url = f\"https://world.openfoodfacts.org/api/v2/product/{product_id}?rev={rev_id}\"\n",
"# Get the product data\n",
"product_data = requests.get(api_url).json()\n",
"product_data"
],
"id": "31a28bb96ee9ff2f",
"outputs": [
{
"data": {
"text/plain": [
"{'code': '875444', 'status': 0, 'status_verbose': 'product not found'}"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": 5
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2024-10-02T05:43:52.597713Z",
"start_time": "2024-10-02T05:43:52.593995Z"
}
},
"cell_type": "code",
"source": "api_url",
"id": "669c29e99544240f",
"outputs": [
{
"data": {
"text/plain": [
"'https://world.openfoodfacts.org/api/v2/product/875444?rev=3'"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": 6
},
{
"metadata": {},
"cell_type": "code",
"outputs": [],
"execution_count": null,
"source": "",
"id": "797b55bb518c5c03"
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading