Skip to content

Commit 60d699f

Browse files
mad-cat-lonmad-cat-lon
committed
Added result caching, rating indicators and new cases
1 parent a6c2149 commit 60d699f

File tree

4 files changed

+175
-41
lines changed

4 files changed

+175
-41
lines changed

‎core/models.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
from pydantic import BaseModel
2-
from typing import Union, List
2+
from typing import Union, List, Dict
33

44

55
class URL(BaseModel):
@@ -14,6 +14,6 @@ class SourceDocument(BaseModel):
1414

1515

1616
class LLMQuery(BaseModel):
17-
tosdr_cases: List[str]
17+
tosdr_cases: List[Dict[str, str]]
1818
service: Union[str, None]
1919
doc_name: Union[str, None]

‎core/server.py

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -219,8 +219,8 @@ async def add_src_document_from_url(urls: List[URL]):
219219
if await scrape_raw_document_from_url(browser, url.url, service):
220220
succeeded += 1
221221
return {
222-
"message": f"Discovered and processed {succeeded}/{len(urls)}\
223-
documents in {service}",
222+
"message": f"Processed {succeeded}/{len(urls)}\
223+
discovered document URLs from {service}",
224224
"service": service
225225
}
226226

@@ -235,13 +235,12 @@ async def make_query(query: LLMQuery):
235235
extension_response = {
236236
"results": []
237237
}
238-
print(query)
239238
# For each case, search the vector database for results
240-
for query_text in query.tosdr_cases:
239+
for q in query.tosdr_cases:
241240
result = {}
242241
query_response = await asyncio.to_thread(
243242
db.similarity_search,
244-
query=query_text,
243+
query=q["text"],
245244
k=4,
246245
filter={"service": query.service},
247246
include=["documents", "metadatas"]
@@ -263,12 +262,12 @@ async def make_query(query: LLMQuery):
263262
]
264263
)
265264
prompt = template.format(
266-
query=query_text,
265+
query=q["text"],
267266
results=[doc.page_content for doc in query_response]
268267
)
269-
# print("="*100)
270-
# print(prompt)
271-
# print("="*100)
268+
print("="*100)
269+
print(prompt)
270+
print("="*100)
272271

273272
llm_response = llm(prompt)
274273
print(llm_response)
@@ -280,7 +279,7 @@ async def make_query(query: LLMQuery):
280279
source_text = chosen_doc.page_content if choice != 0 else ""
281280
# TODO: Fix field duplication later
282281
result["source_text"] = source_text
283-
result["tosdr_case"] = query_text
282+
result["tosdr_case"] = q
284283
result["source_doc"] = chosen_doc.metadata["name"]
285284
result["source_url"] = chosen_doc.metadata["url"]
286285
result["source_service"] = chosen_doc.metadata["service"]

‎extension/public/js/background.js

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,31 @@ function injectGetContent() {
7070
});
7171
}
7272

73+
// function injectGetFavicon() {
74+
// chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
75+
// let currentTabId = tabs[0].id;
76+
// console.log("Current Tab ID: ", currentTabId);
77+
// if (userBrowser == "Chrome") {
78+
// chrome.scripting.executeScript({
79+
// target: { tabId: currentTabId },
80+
// files: ["/js/getFavicon.js"]
81+
// }, () => {
82+
// if (chrome.runtime.lastError) {
83+
// console.error("Error injecting scripts: ", chrome.runtime.lastError);
84+
// }
85+
// });
86+
// }
87+
// else if (userBrowser == "Firefox") {
88+
// browser.tabs.executeScript(currentTabId, { file: '/js/getFavicon.js'})
89+
// .then(() => {
90+
// console.log("Script injected successfully.");
91+
// }).catch((error) => {
92+
// console.error("Error injecting scripts: ", error);
93+
// });
94+
// }
95+
// });
96+
// }
97+
7398
function chunkArray(array, chunkSize) {
7499
return array.reduce((result, item, index) => {
75100
const chunkIndex = Math.floor(index / chunkSize);
@@ -85,6 +110,7 @@ chrome.runtime.onInstalled.addListener(() => {
85110
console.log("Extension initialized.");
86111
});
87112

113+
let resultsCache = {}
88114
let tosdr_cases = [];
89115
const query_endpoint = 'http://127.0.0.1:8000/query';
90116
const url_upload_endpoint = 'http://127.0.0.1:8000/add_from_url';
@@ -246,6 +272,22 @@ chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
246272
// chunk cases so user spends less time waiting
247273
let chunked_cases = chunkArray(tosdr_cases, 3)
248274
let promises = []
275+
if (resultsCache.hasOwnProperty(msg.service)) {
276+
// If we've already analyzed this service before, send the cached results
277+
chrome.runtime.sendMessage({
278+
action: 'updateResults',
279+
data: { 'results': resultsCache[msg.service] }
280+
})
281+
// Tell the user we're giving them the cached results
282+
chrome.runtime.sendMessage({
283+
action: 'backendResponse',
284+
message: 'Returning cached results for' + msg.service
285+
})
286+
}
287+
else {
288+
// Initialize the empty array
289+
resultsCache[msg.service] = []
290+
}
249291
for (let i=0; i<chunked_cases.length; i++) {
250292
let promise = fetch(query_endpoint, {
251293
method: 'POST',
@@ -262,9 +304,10 @@ chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
262304
console.log("Response from backend received: ", data);
263305
chrome.runtime.sendMessage({
264306
action: 'updateResults',
265-
data: data,
266-
source: msg.source,
307+
data: data
267308
});
309+
// Store it in the results log
310+
resultsCache[msg.service].push(...data['results'])
268311
})
269312
.catch(error => {
270313
console.log("Error in fetching: ", error);

‎extension/src/App.js

Lines changed: 119 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -16,52 +16,145 @@ function App() {
1616
const [currentService, setCurrentService] = useState("");
1717
const [results, setResults] = useState([]);
1818
const [theme, setTheme] = useState('light');
19-
19+
20+
const [ratingMap] = useState({
21+
neutral: '📋',
22+
warning: '⚠️',
23+
positive: '✅',
24+
negative: '❌'
25+
})
2026
// Stores tosdr points by category so they can be selected in the menu
2127
const [queryCategories, setQueryCategories] = useState(
2228
[
2329
{
2430
'name': '📒 Account policies',
2531
'cases': [
26-
'This service can delete your account without prior notice and without a reason',
27-
'You must provide your legal name, pseudonyms are not allowed',
28-
'User accounts can be terminated after having been in breach of the terms of service repeatedly'
32+
{
33+
'text': 'This service can delete your account without prior notice and without a reason',
34+
'rating': 'neutral'
35+
},
36+
{
37+
'text': 'You must provide your legal name, pseudonyms are not allowed',
38+
'rating': 'negative'
39+
},
40+
{
41+
'text': 'User accounts can be terminated after having been in breach of the terms of service repeatedly',
42+
'rating': 'neutral'
43+
},
44+
{
45+
'text': 'This service can be used without providing a user profile',
46+
'rating': 'good'
47+
}
2948
],
3049
'checked': false
3150
},
3251
{
3352
'name': '👁️ Tracking and data collection',
3453
'cases': [
35-
'This service tracks you on other websites',
36-
'This service may collect, use, and share location data',
37-
'Third party cookies are used for advertising',
38-
'Tracking via third-party cookies for other purposes without your consent',
39-
'The service may use tracking pixels, web beacons, browser fingerprinting, and/or device fingerprinting on users',
40-
'This service receives your precise location through GPS coordinates',
41-
'Your biometric data is collected',
42-
'This service still tracks you even if you opted out from tracking'
54+
{
55+
'text': 'This service tracks you on other websites',
56+
'rating': 'negative'
57+
},
58+
{
59+
'text': 'This service may collect, use, and share location data',
60+
'rating': 'negative'
61+
},
62+
{
63+
'text': 'Third party cookies are used for advertising',
64+
'rating': 'warning'
65+
},
66+
{
67+
'text': 'Tracking via third-party cookies for other purposes without your consent',
68+
'rating': 'negative'
69+
},
70+
{
71+
'text': 'The service may use tracking pixels, web beacons, browser fingerprinting, and/or device fingerprinting on users',
72+
'rating': 'negative'
73+
},
74+
{
75+
'text': 'This service receives your precise location through GPS coordinates',
76+
'rating': 'negative'
77+
},
78+
{
79+
'text': 'Your biometric data is collected',
80+
'rating': 'negative'
81+
},
82+
{
83+
'text': 'This service still tracks you even if you opted out from tracking',
84+
'rating': 'negative'
85+
},
86+
{
87+
'text': 'The cookies used only collect anonymous, aggregated data that cannot be linked to a unique identity.',
88+
'rating': 'positive'
89+
}
4390
],
4491
'checked': false
4592
},
4693
{
4794
'name': '📁 Data storage and retention',
4895
'cases': [
49-
'This service stores your data whether you have an account or not',
50-
'This service stores data on you even if you did not interact with the service',
51-
'The service can sell or otherwise transfer your personal data as part of a bankruptcy proceeding or other type of financial transaction.',
52-
'This service may keep personal data after a request for erasure for business interests or legal obligations'
96+
{
97+
'text': 'This service stores your data whether you have an account or not',
98+
'rating': 'negative'
99+
},
100+
{
101+
'text': 'This service stores data on you even if you did not interact with the service',
102+
'rating': 'negative'
103+
},
104+
{
105+
'text': 'The service can sell or otherwise transfer your personal data as part of a bankruptcy proceeding or other type of financial transaction.',
106+
'rating': 'negative'
107+
},
108+
{
109+
'text': 'This service may keep personal data after a request for erasure for business interests or legal obligations',
110+
'rating': 'negative'
111+
},
112+
{
113+
'text': 'The service may keep a secure, anonymized record of your data for analytical purposes even after the data retention period',
114+
'rating': 'negative'
115+
}
53116
],
54117
'checked': false
55118
},
56119
{
57120
'name': '⚖️ Legal rights',
58121
'cases': [
59-
'You waive your moral rights',
60-
'This service retains rights to your content even after you stop using your account',
61-
'You waive your right to a class action.',
62-
'You have the right to leave this service at any time',
63-
'You agree to defend, indemnify, and hold the service harmless in case of a claim related to your use of the service',
64-
'This service forces users into binding arbitration in the case of disputes'
122+
{
123+
'text': 'Terms may be changed any time at their discretion, without notice to you',
124+
'rating': 'warning'
125+
},
126+
{
127+
'text': 'You waive your moral rights',
128+
'rating': 'negative'
129+
},
130+
{
131+
'text': 'This service retains rights to your content even after you stop using your account',
132+
'rating': 'negative'
133+
},
134+
{
135+
'text': 'You waive your right to a class action.',
136+
'rating': 'negative'
137+
},
138+
{
139+
'text': 'You have the right to leave this service at any time',
140+
'rating': 'positive'
141+
},
142+
{
143+
'text': 'The service will resist legal requests for your information where reasonably possible',
144+
'rating': 'positive'
145+
},
146+
{
147+
'text': 'You agree to defend, indemnify, and hold the service harmless in case of a claim related to your use of the service',
148+
'rating': 'negative'
149+
},
150+
{
151+
'text': 'This service forces users into binding arbitration in the case of disputes',
152+
'rating': 'negative'
153+
},
154+
{
155+
'text': 'Any liability on behalf of the service is only limited to the fees you paid as a user',
156+
'rating': 'negative'
157+
}
65158
],
66159
'checked': false
67160
}
@@ -82,7 +175,7 @@ function App() {
82175
setIsLoading(false);
83176
setBackendResponse(msg);
84177
setShowAlert(true);
85-
if (msg.error === false && (msg.type === 'upload_content' || msg.type === 'upload_url' || msg.type === 'upload')) {
178+
if (msg.error === false && (msg.type === 'upload_content' || msg.type === 'upload_url')) {
86179
// Handle manual content upload success
87180
// if we successfully uploaded then add to the list of services and docs
88181
setServices(prevServices => {
@@ -177,9 +270,8 @@ function App() {
177270
return;
178271
}
179272

180-
function clearData() {
273+
function clearResults() {
181274
setResults([]);
182-
setServices({});
183275
}
184276

185277
return (
@@ -239,7 +331,7 @@ function App() {
239331

240332
{/* <button className="btn btn-lg btn-secondary btn-wide btn-active w-full" onClick={handleAnalyze}> 👁️ Analyze current page</button> */}
241333
<button className="btn btn-lg btn-secondary btn-wide btn-active w-full" onClick={handleAutoAnalyze}>🔍 Discover documents</button>
242-
<button className="btn btn-lg btn-accent btn-wide btn-active w-full" onClick={clearData}>🗑️ Clear data</button>
334+
<button className="btn btn-lg btn-accent btn-wide btn-active w-full" onClick={clearResults}>🗑️ Clear results</button>
243335
</div>
244336
{
245337
results.length > 0 && (
@@ -254,7 +346,7 @@ function App() {
254346
<div className="collapse collapse-arrow">
255347
<input type="checkbox" />
256348
<div className="collapse-title text-xl font-bold">
257-
⚠️ {item.tosdr_case}
349+
{ratingMap[item.tosdr_case.rating]} {item.tosdr_case.text}
258350
</div>
259351
<div className="collapse-content">
260352
<div className="divider divider-error">

0 commit comments

Comments
 (0)