Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update mocked_http.py #382

Merged
merged 1 commit into from
Sep 3, 2024
Merged

Update mocked_http.py #382

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 73 additions & 48 deletions argopy/tests/helpers/mocked_http.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,7 @@

"""
import contextlib
import json
import os
import sys
from pathlib import Path
import threading
from collections import ChainMap
from http.server import BaseHTTPRequestHandler, HTTPServer
Expand All @@ -36,60 +34,86 @@
from urllib.parse import unquote
import socket
import json
import importlib


log = logging.getLogger("argopy.tests.mocked_http")
LOG_SERVER_CONTENT = False # Also log the list of files/uris available from the mocked server
LOG_SERVER_CONTENT = (
False # Should we list files/uris available from the mocked server in the log ?
)

requests = pytest.importorskip("requests")
port = 9898 # Select the port to run the local server on
mocked_server_address = "http://127.0.0.1:%i" % port

start_with = lambda f, x: f[0:len(x)] == x if len(x) <= len(f) else False # noqa: E731
start_with = (
lambda f, x: f[0 : len(x)] == x if len(x) <= len(f) else False
) # noqa: E731

sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
from argopy.data_fetchers.erddap_data import api_server as erddap_api_server
"""
Load test data and create a dictionary mapping of URL requests as keys, and expected responses as values

# Dictionary mapping of URL requests as keys, and expected responses as values:
# (this will be filling the mocked http server content)
# The real address must be made relative
This will be filling the mocked http server content.
The uri requested will be made relative, because the server name will be replaced by the mocked server address.
"""
MOCKED_REQUESTS = {}
DATA_FOLDER = os.path.dirname(os.path.realpath(__file__)).replace("helpers", "test_data")
DB_FILE = os.path.join(DATA_FOLDER, "httpmocked_uri_index.json")
if os.path.exists(DB_FILE):
TESTDATA_FOLDER = (
Path(importlib.util.find_spec("argopy.tests").submodule_search_locations[0])
.resolve()
.joinpath("test_data")
)
if not TESTDATA_FOLDER.exists():
raise RuntimeError(
"Can't find tests data folder at: %s\n Note that test data are not included in the pypi distribution. You should fork the repo to get test data."
% TESTDATA_FOLDER
)

DB_FILE = TESTDATA_FOLDER.joinpath("httpmocked_uri_index.json")
URI = []

if DB_FILE.exists():
with open(DB_FILE, "r") as f:
URI = json.load(f)
for resource in URI:
test_data_file = os.path.join(DATA_FOLDER, "%s.%s" % (resource['sha'], resource['ext']))
with open(test_data_file, mode='rb') as file:
test_data_file = TESTDATA_FOLDER.joinpath(
"%s.%s" % (resource["sha"], resource["ext"])
)
with open(test_data_file, mode="rb") as file:
data = file.read()

# Remove all specific api/server, that will be served by the mocked http server:
# Remove all specific api/server names from absolute URIs
# Because these are arguments passed to methods that will use mocked_server_address instead
# (See for instance the argument 'server' in `argopy.data_fetchers.erddap_data.ErddapArgoDataFetcher`)
patterns = [
"https://github.com/euroargodev/argopy-data/raw/master",
"https://erddap.ifremer.fr/erddap",
"https://data-argo.ifremer.fr",
"https://api.ifremer.fr",
"https://coastwatch.pfeg.noaa.gov/erddap",
"https://www.ocean-ops.org/api/1",
"https://dataselection.euro-argo.eu/api",
"https://vocab.nerc.ac.uk/collection",
"https://argovisbeta02.colorado.edu",
"https://dx.doi.org",
"https://archimer.ifremer.fr",
"https://github.com/euroargodev/argopy-data/raw/master",
"https://erddap.ifremer.fr/erddap",
"https://data-argo.ifremer.fr",
"https://api.ifremer.fr",
"https://coastwatch.pfeg.noaa.gov/erddap",
"https://www.ocean-ops.org/api/1",
"https://dataselection.euro-argo.eu/api",
"https://vocab.nerc.ac.uk/collection",
"https://argovisbeta02.colorado.edu",
"https://dx.doi.org",
"https://archimer.ifremer.fr",
]
for pattern in patterns:
if start_with(resource['uri'], pattern):
MOCKED_REQUESTS[resource['uri'].replace(pattern, "")] = data
if start_with(resource["uri"], pattern):
MOCKED_REQUESTS[resource["uri"].replace(pattern, "")] = data

else:
log.debug("Loading this sub-module without DB_FILE ! %s" % DB_FILE)
raise RuntimeError(
"Can't find test data index file at: %s.\n Note that test data are not included in the pypi distribution. You should fork the repo to get test data."
% DB_FILE
)


def get_html_landing_page():
"""Return a webpage with a listing of all available files with a href links"""
html = ["<html><head></head><body>\n"]
html.append("<h1>Mocked HTTP server is up and running, serving %i files</h1>" % len(URI))
html.append(
"<h1>Mocked HTTP server is up and running, serving %i files</h1>" % len(URI)
)
html.append("<ul>")
for key, value in MOCKED_REQUESTS.items():
html.append("<li><a href='%s'>%s</a></li>" % (key, key))
Expand All @@ -101,10 +125,6 @@ def get_html_landing_page():
class HTTPTestHandler(BaseHTTPRequestHandler):
static_files = {
"": get_html_landing_page(),
# "": b"Mocked HTTP server is up and running, serving %i files" % len(URI), # This is mandatory for the server to respond without content
# "/index/otherfile": data,
# "/index": index,
# "/data/20020401": listing,
}
dynamic_files = {}

Expand Down Expand Up @@ -150,25 +170,26 @@ def do_GET(self):
if file_data is None:
# log.debug("file data empty, returning 404")
return self._respond(404)
else:
n = len(file_data)

status = 200
content_range = "bytes 0-%i/%i" % (len(file_data) - 1, len(file_data))
content_range = "bytes 0-%i/%i" % (n - 1, n)
if ("Range" in self.headers) and ("ignore_range" not in self.headers):
ran = self.headers["Range"]
b, ran = ran.split("=")
start, end = ran.split("-")
if start:
content_range = f"bytes {start}-{end}/{len(file_data)}"
file_data = file_data[int(start): (int(end) + 1) if end else None]
content_range = f"bytes {start}-{end}/{n}"
file_data = file_data[int(start) : (int(end) + 1) if end else None]
else:
# suffix only
l = len(file_data)
content_range = f"bytes {l - int(end)}-{l - 1}/{l}"
file_data = file_data[-int(end):]
content_range = f"bytes {n - int(end)}-{n - 1}/{n}"
file_data = file_data[-int(end) :]
if "use_206" in self.headers:
status = 206
if "give_length" in self.headers:
response_headers = {"Content-Length": len(file_data)}
response_headers = {"Content-Length": n}
self._respond(status, response_headers, file_data)
elif "give_range" in self.headers:
self._respond(status, {"Content-Range": content_range}, file_data)
Expand Down Expand Up @@ -199,8 +220,8 @@ def read_chunks(self):
self.rfile.readline()

def do_HEAD(self):
self.headers.add_header('head_ok', '')
self.headers.add_header('give_length', '')
self.headers.add_header("head_ok", "")
self.headers.add_header("give_length", "")

if "head_not_auth" in self.headers:
return self._respond(
Expand All @@ -213,16 +234,18 @@ def do_HEAD(self):
file_data = self.files.get(file_path)
if file_data is None:
return self._respond(404)
else:
n = len(file_data)

if ("give_length" in self.headers) or ("head_give_length" in self.headers):
response_headers = {"Content-Length": len(file_data)}
response_headers = {"Content-Length": n}
if "zero_length" in self.headers:
response_headers["Content-Length"] = 0

self._respond(200, response_headers)
elif "give_range" in self.headers:
self._respond(
200, {"Content-Range": "0-%i/%i" % (len(file_data) - 1, len(file_data))}
200, {"Content-Range": "0-%i/%i" % (n - 1, n)}
)
elif "give_etag" in self.headers:
self._respond(200, {"ETag": "xxx"})
Expand All @@ -238,8 +261,10 @@ def serve_mocked_httpserver():
th.daemon = True
th.start()
try:
log.info("Mocked HTTP server up and ready at %s, serving %i URI. (id=%s)" %
(mocked_server_address, len(HTTPTestHandler.files), id(httpd)))
log.info(
"Mocked HTTP server up and ready at %s, serving %i URI. (id=%s)"
% (mocked_server_address, len(HTTPTestHandler.files), id(httpd))
)
if LOG_SERVER_CONTENT:
# Use these lines to log test data name and content
for f in HTTPTestHandler.files.keys():
Expand Down