From 543ba122b551333e2cf4c6c18b70988daedafa15 Mon Sep 17 00:00:00 2001 From: aspedrosa Date: Thu, 13 Jan 2022 19:43:25 +0000 Subject: [PATCH 1/6] clean some code --- geowebservice/Dockerfile | 11 +- geowebservice/all.json | 0 geowebservice/build.sh | 1 - geowebservice/docker-compose.yml | 24 +-- geowebservice/geodatabase/admin.py | 11 -- .../management/commands/load_data.py | 26 +-- geowebservice/geodatabase/models.py | 64 ------- geowebservice/geodatabase/services.py | 180 ++++++------------ geowebservice/geodatabase/tests.py | 16 -- geowebservice/geodatabase/utils.py | 22 ++- geowebservice/geodatabase/views.py | 171 ++++++++--------- geowebservice/geowebservice/settings.py | 35 +--- geowebservice/geowebservice/urls.py | 21 +- geowebservice/geowebservice/wsgi.py | 6 +- geowebservice/import_countryInfo.py | 26 --- geowebservice/load_data.py | 65 ------- geowebservice/requirements.txt | 2 +- geowebservice/sql/geonames_countryInfo.sql | 50 ----- geowebservice/sql/geonames_geoname.sql | 53 ------ geowebservice/sqlite3.db | Bin 46080 -> 0 bytes geowebservice/start.sh | 1 - 21 files changed, 193 insertions(+), 592 deletions(-) delete mode 100644 geowebservice/all.json delete mode 100644 geowebservice/build.sh delete mode 100644 geowebservice/geodatabase/admin.py delete mode 100644 geowebservice/geodatabase/models.py delete mode 100644 geowebservice/geodatabase/tests.py delete mode 100644 geowebservice/import_countryInfo.py delete mode 100644 geowebservice/load_data.py delete mode 100644 geowebservice/sql/geonames_countryInfo.sql delete mode 100644 geowebservice/sql/geonames_geoname.sql delete mode 100644 geowebservice/sqlite3.db delete mode 100644 geowebservice/start.sh diff --git a/geowebservice/Dockerfile b/geowebservice/Dockerfile index b1a41a3..e74910b 100644 --- a/geowebservice/Dockerfile +++ b/geowebservice/Dockerfile @@ -1,12 +1,11 @@ -FROM debian:jessie +FROM debian:stretch MAINTAINER Matt Bentley -RUN (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential git python python-dev python-setuptools nginx sqlite3 supervisor) -RUN (easy_install pip &&\ - pip install uwsgi) +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -y build-essential python python-dev python-setuptools nginx sqlite3 supervisor python-pip ADD requirements.txt /opt/django/app/requirements.txt -RUN pip install -r /opt/django/app/requirements.txt +RUN pip install -r /opt/django/app/requirements.txt && pip install uwsgi ADD . /opt/django/ RUN (echo "daemon off;" >> /etc/nginx/nginx.conf &&\ @@ -17,4 +16,4 @@ RUN (echo "daemon off;" >> /etc/nginx/nginx.conf &&\ RUN chmod +x /opt/django/run.sh VOLUME ["/opt/django/app"] EXPOSE 80 -CMD ["/opt/django/run.sh"] \ No newline at end of file +CMD ["/opt/django/run.sh"] diff --git a/geowebservice/all.json b/geowebservice/all.json deleted file mode 100644 index e69de29..0000000 diff --git a/geowebservice/build.sh b/geowebservice/build.sh deleted file mode 100644 index 0b1e8e3..0000000 --- a/geowebservice/build.sh +++ /dev/null @@ -1 +0,0 @@ - docker build --rm=true -t bioinformaticsua/geodropdownservice . \ No newline at end of file diff --git a/geowebservice/docker-compose.yml b/geowebservice/docker-compose.yml index f3e5885..f174a36 100644 --- a/geowebservice/docker-compose.yml +++ b/geowebservice/docker-compose.yml @@ -1,17 +1,17 @@ -solr: +version: "3" + +services: + solr: image: solr:5.5.3 - hostname: solr ports: - - "8887:8983" - - + - 8983:8983 -geodropdownservice: - image: bioinformaticsua/geodropdownservice + geodropdownservice: + build: + context: . ports: - - "8886:80" + - "8886:80" volumes: - - .:/opt/django/app - links: - - solr:solr - + - .:/opt/django/app + depends_on: + - solr diff --git a/geowebservice/geodatabase/admin.py b/geowebservice/geodatabase/admin.py deleted file mode 100644 index 6f9e412..0000000 --- a/geowebservice/geodatabase/admin.py +++ /dev/null @@ -1,11 +0,0 @@ -from geodatabase.models import Geoname,Countryinfo -from django.contrib import admin - -class GeonameAdmin(admin.ModelAdmin): - list_display = ('geonameid', 'asciiname', 'fcode', 'country', 'admin1', 'admin2', 'admin3', 'admin4') - -class CountryinfoAdmin(admin.ModelAdmin): - list_display = ('name', 'continent', 'geonameId') - -admin.site.register(Geoname,GeonameAdmin) -admin.site.register(Countryinfo,CountryinfoAdmin) diff --git a/geowebservice/geodatabase/management/commands/load_data.py b/geowebservice/geodatabase/management/commands/load_data.py index e376030..97c300d 100644 --- a/geowebservice/geodatabase/management/commands/load_data.py +++ b/geowebservice/geodatabase/management/commands/load_data.py @@ -1,18 +1,14 @@ # -*- coding: utf-8 -*- -from __future__ import print_function -import pysolr - -from django.core.management.base import BaseCommand, CommandError +import logging -import ast -from django.conf import settings +from django.core.management.base import BaseCommand from geodatabase.services import ServiceSolr -import logging logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG) + class Command(BaseCommand): help = 'Closes the specified poll for voting' @@ -21,15 +17,9 @@ def add_arguments(self, parser): parser.add_argument('countries', nargs='+', type=str) def handle(self, *args, **options): - all_locations = '' - countries_names = '' - try: - all_locations = options['all'][0] - countries_names = options['countries'][0] - except: - all_locations = '/home/leonardo/Área de Trabalho/geonames_files/allCountries.txt' - countries_names = '/home/leonardo/Área de Trabalho/geonames_files/country.csv' - + all_locations = options['all'][0] + countries_names = options['countries'][0] + s = ServiceSolr() - s.load_contry_info(countries_names) - s.load_initial_data(all_locations) \ No newline at end of file + s.load_country_info(countries_names) + s.load_initial_data(all_locations) diff --git a/geowebservice/geodatabase/models.py b/geowebservice/geodatabase/models.py deleted file mode 100644 index 51eb732..0000000 --- a/geowebservice/geodatabase/models.py +++ /dev/null @@ -1,64 +0,0 @@ -# This is an auto-generated Django model module. -# You'll have to do the following manually to clean this up: -# * Rearrange models' order -# * Make sure each model has one field with primary_key=True -# Feel free to rename the models, but don't rename db_table values or field names. -# -# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]' -# into your database. - -from django.db import models - - - -class Countryinfo(models.Model): - iso_alpha2 = models.CharField(max_length=765, blank=True) - iso_alpha3 = models.CharField(max_length=765, blank=True) - iso_numeric = models.IntegerField(null=True, blank=True) - fips_code = models.CharField(max_length=900, blank=True) - name = models.CharField(max_length=6000, blank=True) - capital = models.CharField(max_length=6000, blank=True) - areainsqkm = models.FloatField(null=True, blank=True) - population = models.IntegerField(null=True, blank=True) - continent = models.CharField(max_length=765, blank=True) - currency = models.CharField(max_length=765, blank=True) - currencyName = models.CharField(max_length=765, db_column='currencyName', blank=True) - geonameId = models.IntegerField(primary_key=True,db_column='geonameId', blank=True) - class Meta: - db_table = u'countryInfo' - - def __unicode__(self): - return str(self.geonameId)+','+self.name+','+self.continent - -class Geoname(models.Model): - geonameid = models.IntegerField(primary_key=True,) - name = models.CharField(max_length=600, blank=True) - asciiname = models.CharField(max_length=600, blank=True) - alternatenames = models.CharField(max_length=12000, blank=True) - latitude = models.DecimalField(null=True, max_digits=12, decimal_places=7, blank=True) - longitude = models.DecimalField(null=True, max_digits=12, decimal_places=7, blank=True) - fclass = models.CharField(max_length=3, blank=True) - fcode = models.CharField(max_length=30, blank=True) - country = models.CharField(max_length=6, blank=True) - cc2 = models.CharField(max_length=180, blank=True) - admin1 = models.CharField(max_length=60, blank=True) - admin2 = models.CharField(max_length=240, blank=True) - admin3 = models.CharField(max_length=60, blank=True) - admin4 = models.CharField(max_length=60, blank=True) - - class Meta: - db_table = u'geoname' - - def __unicode__(self): - selfName = str(self.geonameid)+'\t'+self.asciiname+'\t'+self.fcode - if self.country: - selfName += '\t'+str(self.country) - if self.admin1: - selfName += '\t'+str(self.admin1) - if self.admin2: - selfName += '\t'+str(self.admin2) - if self.admin3: - selfName += '\t'+str(self.admin3) - if self.admin4: - selfName += '\t'+str(self.admin4) - return selfName \ No newline at end of file diff --git a/geowebservice/geodatabase/services.py b/geowebservice/geodatabase/services.py index e42419c..db8e6da 100644 --- a/geowebservice/geodatabase/services.py +++ b/geowebservice/geodatabase/services.py @@ -18,135 +18,82 @@ """ from __future__ import print_function -import pysolr - +import logging -import ast +import pysolr from django.conf import settings -import csv -import random - -import logging +from geodatabase.utils import UnicodeReader -from utils import * logger = logging.getLogger() - logging.basicConfig(level=logging.DEBUG) -"""convert input to unicode""" -def make_unicode(input): - if type(input) != unicode: - input = input.decode('utf-8') - return input - else: - return input - -""" -Search service, that contains all the information about contries. -It also has methods to load the initial data and to update them -""" class ServiceSolr(object): + """ + Search service, that contains all the information about contries. + It also has methods to load the initial data and to update them + """ + CONNECTION_TIMEOUT_DEFAULT = 99000000 - - def __fetch_initial_settings(self): - try: - self.SOLR_HOST = settings.SOLR_HOST - self.SOLR_PORT = settings.SOLR_PORT - self.SOLR_PATH = settings.SOLR_PATH - except: - # TODO: - # Maybe should load default settings here? - logger.error("It is not running in Django enviroment") - raise - def __init__(self, host="solr", port="8983", path="/solr", timeout=CONNECTION_TIMEOUT_DEFAULT, core="geonames"): + + def __init__(self): # Setup a Solr instance. The timeout is optional. logger.info("Initial Solr Service") - try: - self.__fetch_initial_settings() - except: - self.SOLR_HOST = "localhost" - self.SOLR_PORT = "8983" - self.SOLR_PATH = "/solr" - - self.solr = pysolr.Solr('http://' +self.SOLR_HOST+ ':'+ self.SOLR_PORT+self.SOLR_PATH+'/', timeout=timeout) - logger.info("Connected to Solr") - - def load_contry_info(self, countryFile="country.csv"): - self.contryInfo = {} - - import csv - ifile = open(countryFile, "rU") - #reader = csv.reader(ifile, delimiter='\t') - reader = UnicodeReader(ifile, delimiter='\t', ) - for row in reader: - - #a.name = row[4] - # a.continent = row[8] - # a.geonameId = row[11] - self.contryInfo[row[11]] = {'continent': row[8],'name': row[4]} - - - + + self.solr = pysolr.Solr( + 'http://{}:{}/solr/{}/'.format( + settings.SOLR_HOST, + settings.SOLR_PORT, + settings.SOLR_CORE, + ), + timeout=self.CONNECTION_TIMEOUT_DEFAULT, + ) + logger.info("Connected to Solr") + + def load_country_info(self, countryFile): + with open(countryFile, "rU") as ifile: + reader = UnicodeReader(ifile, delimiter='\t', ) + + self.countryInfo = {row[11]: {'continent': row[8], 'name': row[4]} for row in reader} + def load_initial_data(self, allCountriesFile): logger.info("Load initial data to Solr") - + with open(allCountriesFile) as csvfile: - #spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') spamreader = UnicodeReader(csvfile, delimiter='\t', quotechar='|') list_docs_to_commit = [] - i = 0 + i = 0 for row in spamreader: - if not (row[1]=='Earth' or row[7] == 'CONT' or row[7] == 'PCLI' or row[7] == 'ISLS' or row[7] == 'ADM1' or row[7] == 'ADM2' or row[7] == 'ADM3' or row[7] == 'ADM4'): + if not (row[1] == 'Earth' or row[7] == 'CONT' or row[7] == 'PCLI' or row[7] == 'ISLS' or + row[7] == 'ADM1' or row[7] == 'ADM2' or row[7] == 'ADM3' or row[7] == 'ADM4'): continue i = i + 1 - d = {} - d['id'] = i - d['geonameId_t'] = row[0] - print(row[0]) - d['name_t'] = row[1] - d['asciiname_t'] = row[2] - d['alternatenames_t'] = row[3] - d['latitude_f'] = row[4] - d['longitude_f'] = row[5] - d['fclass_t'] = row[6] - d['fcode_t'] = row[7] - try: - d['country_t'] = row[8] - except: - pass - d['cc2_t'] = row[9] - d['admin1_t'] = row[10] - d['admin2_t'] = row[11] - d['admin3_t'] = row[12] - d['admin4_t'] = row[13] - d['population_t'] = row[14] - d['elevation_t'] = row[15] - d['gtopo30_t'] = row[16] - d['timezone_t'] = row[17] - - if row[0] in self.contryInfo: - d['continent_t'] = self.contryInfo[row[0]]['continent'] - d['name_t'] = self.contryInfo[row[0]]['name'] - - #print(d) + d = {'id': i, 'geonameId_t': row[0], 'name_t': row[1], 'asciiname_t': row[2], + 'alternatenames_t': row[3], 'latitude_f': row[4], 'longitude_f': row[5], 'fclass_t': row[6], + 'fcode_t': row[7], 'country_t': row[8], 'cc2_t': row[9], 'admin1_t': row[10], 'admin2_t': row[11], + 'admin3_t': row[12], + 'admin4_t': row[13], 'population_t': row[14], 'elevation_t': row[15], 'gtopo30_t': row[16], + 'timezone_t': row[17]} + + if row[0] in self.countryInfo: + d['continent_t'] = self.countryInfo[row[0]]['continent'] + d['name_t'] = self.countryInfo[row[0]]['name'] + d = dict(d.items()) list_docs_to_commit.append(d) - if (i%60000==0): - xml_answer = self.solr.add(list_docs_to_commit) + if i % 60000 == 0: + self.solr.add(list_docs_to_commit) list_docs_to_commit = [] - print(i) - - xml_answer = self.solr.add(list_docs_to_commit) - list_docs_to_commit = [] + print(i) + + self.solr.add(list_docs_to_commit) self.solr.optimize() - - """Fetch the result by id""" + def geonameId(self, geonameId, start=0, rows=100, fl='', sort='', facet="off"): - print(self.solr) - results = self.solr.search("geonameId_t:"+geonameId,**{ + """Fetch the result by id""" + results = self.solr.search("geonameId_t:" + geonameId, **{ 'facet': facet, 'rows': rows, 'start': start, @@ -154,12 +101,12 @@ def geonameId(self, geonameId, start=0, rows=100, fl='', sort='', facet="off"): 'sort': sort }) return results - - - """Generic search - """ + def search(self, query, start=0, rows=500, fl='', sort='', facet="off"): - results = self.solr.search(query,**{ + """ + Generic search + """ + results = self.solr.search(query, **{ 'facet': facet, 'rows': rows, 'start': start, @@ -167,20 +114,3 @@ def search(self, query, start=0, rows=500, fl='', sort='', facet="off"): 'sort': sort }) return results - - -def main(): - s = ServiceSolr() - allCountriesFile = '/opt/django/app/allCountries.txt' - allCountriesFile = '../allCountries.txt' - countryFile = '/opt/django/app/country.csv' - countryFile = '../country.csv' - print(allCountriesFile) - s.load_contry_info(countryFile) - s.load_initial_data(allCountriesFile) - - #results = s.search("3039162") - #d = results.docs[0] - #print(d) -if __name__ == "__main__": - main() diff --git a/geowebservice/geodatabase/tests.py b/geowebservice/geodatabase/tests.py deleted file mode 100644 index 501deb7..0000000 --- a/geowebservice/geodatabase/tests.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -This file demonstrates writing tests using the unittest module. These will pass -when you run "manage.py test". - -Replace this with more appropriate tests for your application. -""" - -from django.test import TestCase - - -class SimpleTest(TestCase): - def test_basic_addition(self): - """ - Tests that 1 + 1 always equals 2. - """ - self.assertEqual(1 + 1, 2) diff --git a/geowebservice/geodatabase/utils.py b/geowebservice/geodatabase/utils.py index e962ef2..696a091 100644 --- a/geowebservice/geodatabase/utils.py +++ b/geowebservice/geodatabase/utils.py @@ -15,39 +15,51 @@ # along with this program. If not, see . # From http://stackoverflow.com/questions/17245415/read-and-write-csv-files-including-unicode-with-python-2-7 -import csv,codecs,cStringIO +import cStringIO +import codecs +import csv + class UTF8Recoder: def __init__(self, f, encoding): self.reader = codecs.getreader(encoding)(f) + def __iter__(self): return self + def next(self): return self.reader.next().encode("utf-8") + class UnicodeReader: def __init__(self, f, dialect=csv.excel, encoding="utf-8-sig", **kwds): f = UTF8Recoder(f, encoding) self.reader = csv.reader(f, dialect=dialect, **kwds) + def next(self): - '''next() -> unicode + """ + next() -> unicode This function reads and returns the next line as a Unicode string. - ''' + """ row = self.reader.next() return [unicode(s, "utf-8") for s in row] + def __iter__(self): return self + class UnicodeWriter: def __init__(self, f, dialect=csv.excel, encoding="utf-8-sig", **kwds): self.queue = cStringIO.StringIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)() + def writerow(self, row): - '''writerow(unicode) -> None + """ + writerow(unicode) -> None This function takes a Unicode string and encodes it to the output. - ''' + """ self.writer.writerow([s.encode("utf-8") for s in row]) data = self.queue.getvalue() data = data.decode("utf-8") diff --git a/geowebservice/geodatabase/views.py b/geowebservice/geodatabase/views.py index 0333f98..b24427c 100644 --- a/geowebservice/geodatabase/views.py +++ b/geowebservice/geodatabase/views.py @@ -15,146 +15,145 @@ # along with this program. If not, see . # Create your views here. import json -from django.http import HttpResponse -from geodatabase.models import Geoname,Countryinfo -from django.http import HttpResponseNotAllowed, HttpResponseBadRequest, HttpResponseNotFound, HttpResponse, HttpResponseForbidden +import logging -from geodatabase.services import * +from django.http import HttpResponseBadRequest, HttpResponse +from .services import ServiceSolr -import logging logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG) -# TODO: a refactor on that method need to be done. + +# TODO: a refactor on that method need to be done. # The business logic should not leave here! -"""This method handles with all request for all hierarchies. Thus, depending on the id, -it will analyse what type of location that it handles and then returns -the proper locations according to the hierarchy. -""" + def detail(request, geonameid): - + """ + This method handles with all request for all hierarchies. Thus, depending on the id, + it will analyse what type of location that it handles and then returns + the proper locations according to the hierarchy. + """ + # TODO: validate geonameid properly. - if geonameid!="0": + if geonameid != "0": solr = ServiceSolr() results = solr.geonameId(geonameid) - + # Check it, if no results in the geonames database, it returns an error - if (len(results)==0): + if len(results) == 0: return HttpResponseBadRequest() - + # Fetch one, and only one document # d variable corresponding now to the geoNameId that is passed as parameter d = results.docs[0] - + # Fetch the data to variables, only to facilitate the access and to become the code easy to read. fcode = d['fcode_t'] name = d['name_t'] - + try: country = d['country_t'] except: pass - + try: admin1 = d['admin1_t'] except: pass - + try: admin2 = d['admin2_t'] except: pass - + try: admin3 = d['admin3_t'] except: pass - + try: admin4 = d['admin4_t'] except: pass else: - name = "Mundus" + name = "Mundus" response_data = [] if name == 'Earth': response_object = solr.search("fcode_t:CONT") - response_data = buildJson(response_object.docs,response_data) - + response_data = buildJson(response_object.docs, response_data) + elif name == 'Mundus': solr = ServiceSolr() - response_object = solr.search("continent_t:EU OR continent_t:AF OR continent_t:OC OR continent_t:SA OR continent_t:NA OR continent_t:AS") - response_data = buildJson(response_object.docs,response_data) + response_object = solr.search( + "continent_t:EU OR continent_t:AF OR continent_t:OC OR continent_t:SA OR continent_t:NA OR continent_t:AS") + response_data = buildJson(response_object.docs, response_data) elif fcode == 'CONT': if name == 'Europe': tmp = solr.search("continent_t:EU") - response_data = buildJson(tmp.docs,response_data) - + response_data = buildJson(tmp.docs, response_data) + elif name == 'Africa': - tmp = solr.search("continent_t:AF") - response_data = buildJson(tmp.docs,response_data) - + response_data = buildJson(tmp.docs, response_data) + elif name == 'Oceania': tmp = solr.search("continent_t:OC") - response_data = buildJson(tmp.docs,response_data) - + response_data = buildJson(tmp.docs, response_data) + elif name == 'South America': tmp = solr.search("continent_t:SA") - response_data = buildJson(tmp.docs,response_data) - + response_data = buildJson(tmp.docs, response_data) + elif name == 'North America': tmp = solr.search("continent_t:NA") - response_data = buildJson(tmp.docs,response_data) - + response_data = buildJson(tmp.docs, response_data) + elif name == 'Asia': tmp = solr.search("continent_t:AS") - response_data = buildJson(tmp.docs,response_data) + response_data = buildJson(tmp.docs, response_data) elif fcode == 'PCLI': - #response_object = Geoname.objects.filter(country=location[0].country,fcode='ADM1') - response_object = solr.search("country_t:"+country+" AND (fcode_t:ADM1 OR fcode_t:ISLS)") - response_data = buildJson(response_object.docs,response_data) + response_object = solr.search("country_t:" + country + " AND (fcode_t:ADM1 OR fcode_t:ISLS)") + response_data = buildJson(response_object.docs, response_data) elif fcode == 'ADM1' or fcode == 'ISLS': - response_object = solr.search("country_t:"+country+" AND admin1_t:"+admin1+" AND fcode_t:ADM2") - #response_object = Geoname.objects.filter(country=location[0].country,admin1=location[0].admin1,fcode='ADM2') - response_data=buildJson(response_object.docs,response_data) + response_object = solr.search("country_t:" + country + " AND admin1_t:" + admin1 + " AND fcode_t:ADM2") + response_data = buildJson(response_object.docs, response_data) elif fcode == 'ADM2': - #response_object = Geoname.objects.filter(country=location[0].country,admin1=location[0].admin1,admin2=location[0].admin2,fcode='ADM3') - response_object = solr.search("country_t:"+country+" AND admin1_t:"+admin1+" AND admin2_t:"+admin2+" AND fcode_t:ADM3") - response_data = buildJson(response_object.docs,response_data) + response_object = solr.search( + "country_t:" + country + " AND admin1_t:" + admin1 + " AND admin2_t:" + admin2 + " AND fcode_t:ADM3") + response_data = buildJson(response_object.docs, response_data) elif fcode == 'ADM3': - #response_object = Geoname.objects.filter(country=location[0].country,admin1=location[0].admin1,admin2=location[0].admin2,admin3=location[0].admin3,fcode='ADM4') - response_object = solr.search("country_t:"+country+" AND admin1_t:"+admin1+" AND admin2_t:"+admin2+" AND admin3_t:"+admin3+" AND fcode_t:ADM4") - - response_data = buildJson(response_object.docs,response_data) + response_object = solr.search( + "country_t:" + country + " AND admin1_t:" + admin1 + " AND admin2_t:" + admin2 + " AND admin3_t:" + admin3 + " AND fcode_t:ADM4") + + response_data = buildJson(response_object.docs, response_data) elif fcode == 'ADM4': - #response_object = Geoname.objects.filter(country=location[0].country,admin1=location[0].admin1,admin2=location[0].admin2,admin3=location[0].admin3,admin4=location[0].admin4,fcode='ADM5') - response_object = solr.search("country_t:"+country+" AND admin1_t:"+admin1+" AND admin2_t:"+admin2+" AND admin3_t:"+admin3+" AND admin4_t:"+admin4+" AND fcode_t:ADM5") - - response_data = buildJson(response_object.docs,response_data) + response_object = solr.search( + "country_t:" + country + " AND admin1_t:" + admin1 + " AND admin2_t:" + admin2 + " AND admin3_t:" + admin3 + " AND admin4_t:" + admin4 + " AND fcode_t:ADM5") + + response_data = buildJson(response_object.docs, response_data) - return HttpResponse(json.dumps(response_data),content_type="application/json") + return HttpResponse(json.dumps(response_data), content_type="application/json") -""" Builds the answer with multiples geonames entries""" -def buildJson(response_object,response_data): - for i in range(0,len(response_object)-1): +def buildJson(response_object, response_data): + """ Builds the answer with multiples geonames entries""" + for i in range(0, len(response_object) - 1): print(response_object[i]) - response_data = response_data+addEntry(response_object[i],response_data) + response_data = response_data + addEntry(response_object[i], response_data) return response_data -""" -Handles a GeoName entry -""" -def addEntry(geoname,response_data): - response = {} +def addEntry(geoname, response_data): + """ + Handles a GeoName entry + """ + response = {} print(geoname) response['geonameid'] = int(geoname['geonameId_t']) response['name'] = geoname['name_t'] @@ -167,34 +166,36 @@ def addEntry(geoname,response_data): response['adm4'] = geoname['admin3_t'] except: pass - + return [response] -""" -Get coordinates by location name and fcode -""" -def getCoordinates(request,location): - if location == '' or location == None: - return HttpResponse(json.dumps([]),content_type="application/json") + +def getCoordinates(request, location): + """ + Get coordinates by location name and fcode + """ + if location == '' or location is None: + return HttpResponse(json.dumps([]), content_type="application/json") location = location.strip().split(',') - levels = [ 'PCLI', 'ADM1', 'ADM2', 'ADM3', 'ADM4', 'ADM5' ] - fcode = levels[len(location)-1] + levels = ['PCLI', 'ADM1', 'ADM2', 'ADM3', 'ADM4', 'ADM5'] + fcode = levels[len(location) - 1] name = location[0] - + try: solr = ServiceSolr() - response_object = solr.search("fcode_t:"+fcode+" AND name_t:"+name) + response_object = solr.search("fcode_t:" + fcode + " AND name_t:" + name) if fcode == 'ADM1' and response_object.hits == 0: - response_object = solr.search("fcode_t:ISLS AND name_t:"+name) + response_object = solr.search("fcode_t:ISLS AND name_t:" + name) response_object = response_object.docs[0] - response_data = buildCoordinates(response_object,[]) - return HttpResponse(json.dumps(response_data),content_type="application/json") + response_data = buildCoordinates(response_object, []) + return HttpResponse(json.dumps(response_data), content_type="application/json") except: - return HttpResponse(json.dumps([]),content_type="application/json") + return HttpResponse(json.dumps([]), content_type="application/json") + -""" -Build a list with latitude and longitude of the location -""" -def buildCoordinates(response_object,response_data): - return [ response_object['latitude_f'], response_object['longitude_f'] ] \ No newline at end of file +def buildCoordinates(response_object, response_data): + """ + Build a list with latitude and longitude of the location + """ + return [response_object['latitude_f'], response_object['longitude_f']] diff --git a/geowebservice/geowebservice/settings.py b/geowebservice/geowebservice/settings.py index b13b207..aad1363 100644 --- a/geowebservice/geowebservice/settings.py +++ b/geowebservice/geowebservice/settings.py @@ -8,20 +8,6 @@ MANAGERS = ADMINS - - - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'geonames', - 'USER': 'root', - 'PASSWORD': 'root', - 'HOST': 'hs', - 'PORT': '3306', - } -} - DATABASES = { 'default' : { 'ENGINE' :'django.db.backends.sqlite3', @@ -29,11 +15,6 @@ } } -TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', - # 'django.template.loaders.eggs.Loader', -) # Hosts/domain names that are valid for this site; required if DEBUG is False # See https://docs.djangoproject.com/en/1.4/ref/settings/#allowed-hosts @@ -103,7 +84,6 @@ TEMPLATE_LOADERS = ( 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', -# 'django.template.loaders.eggs.Loader', ) MIDDLEWARE_CLASSES = ( @@ -129,20 +109,15 @@ ) INSTALLED_APPS = ( - - 'django.contrib.admin', - 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', - # Uncomment the next line to enable the admin: + 'corsheaders', 'geodatabase', - # Uncomment the next line to enable admin documentation: - # 'django.contrib.admindocs', ) # A sample logging configuration. The only tangible logging @@ -174,10 +149,8 @@ } } -SOLR_HOST = "localhost" +SOLR_HOST = "solr" SOLR_PORT = "8983" -SOLR_PATH = "/solr" - - +SOLR_CORE = "geonames" -CORS_ORIGIN_ALLOW_ALL = True \ No newline at end of file +CORS_ORIGIN_ALLOW_ALL = True diff --git a/geowebservice/geowebservice/urls.py b/geowebservice/geowebservice/urls.py index 676a8f6..2ad0bd9 100644 --- a/geowebservice/geowebservice/urls.py +++ b/geowebservice/geowebservice/urls.py @@ -1,24 +1,11 @@ -from django.conf.urls import patterns, include, url +from django.conf.urls import patterns, url -# Uncomment the next two lines to enable the admin: -from django.contrib import admin -admin.autodiscover() - -urlpatterns = patterns('', - # Examples: - # url(r'^$', '{{ project_name }}.views.home', name='home'), - # url(r'^{{ project_name }}/', include('{{ project_name }}.foo.urls')), - # url(r'^geodatabase/$', 'geodatabase.views.index'), +urlpatterns = patterns( + "", # get child locations by geonameid url(r'^geodatabase/(?P\d+)/$', 'geodatabase.views.detail'), # get coordinates by name and fcode url(r'^geodatabase/(?P[a-zA-Z, ]+)/$', 'geodatabase.views.getCoordinates'), - - # Uncomment the admin/doc line below to enable admin documentation: - # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), - - # Uncomment the next line to enable the admin: - url(r'^admin/', include(admin.site.urls)), -) \ No newline at end of file +) diff --git a/geowebservice/geowebservice/wsgi.py b/geowebservice/geowebservice/wsgi.py index f20f185..a924bfb 100644 --- a/geowebservice/geowebservice/wsgi.py +++ b/geowebservice/geowebservice/wsgi.py @@ -14,15 +14,11 @@ """ import os +from django.core.wsgi import get_wsgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geowebservice.settings") # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. -from django.core.wsgi import get_wsgi_application application = get_wsgi_application() - -# Apply WSGI middleware here. -# from helloworld.wsgi import HelloWorldApplication -# application = HelloWorldApplication(application) diff --git a/geowebservice/import_countryInfo.py b/geowebservice/import_countryInfo.py deleted file mode 100644 index 5052671..0000000 --- a/geowebservice/import_countryInfo.py +++ /dev/null @@ -1,26 +0,0 @@ - - - -import csv - -from geodatabase.models import Countryinfo - -import csv -ifile = open("country.csv", "rU") -reader = csv.reader(ifile, delimiter='\t') -for row in reader: - print row - a = Countryinfo() - a.iso_alpha2 = row[0] - a.iso_alpha3 = row[1] - a.iso_numeric = row[2] - a.fips_code = row[3] - a.name = row[4] - a.capital = row[5] - a.areainsqkm = row[6] - a.population = row[7] - a.continent = row[8] - a.currency = row[9] - a.currencyName = row[10] - a.geonameId = row[11] - a.save() \ No newline at end of file diff --git a/geowebservice/load_data.py b/geowebservice/load_data.py deleted file mode 100644 index 7157665..0000000 --- a/geowebservice/load_data.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - - -from geodatabase.models import Geoname -from django.db import transaction - -import csv - -i = 0 -def make_unicode(input): - if type(input) != unicode: - input = input.decode('utf-8') - return input - else: - return input -with open('/Users/bastiao/Downloads/allCountries.txt') as csvfile: - spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') - l = [] - - for row in spamreader: - i = i + 1 - - g = Geoname() - try: - g.geonameid = int(row[0]) - except: - pass - try: - g.name = make_unicode(row[1]) - except: - pass - try: - - g.asciiname = make_unicode(row[2]) - #g.alternatenames = make_unicode(row[3]) - except: - pass - - g.latitude = float(row[4]) - g.longitude = float(row[5]) - g.fclass = row[6] - g.country = row[7] - g.cc2 = row[8] - g.admin1 = row[9] - g.admin2 = row[10] - g.admin3 = row[11] - g.admin4 = row[12] - try: - g.population = int(row[13]) - g.elevation = int(row[14]) - g.gtopo30 = int(row[15]) - g.timezone = row[16] - except: - pass - #g.moddate = Date(row[17]) - if (i%6000==0): - print i - Geoname.objects.bulk_create(l) - l = [] - else: - l.append(g) - Geoname.objects.bulk_create(l) - - \ No newline at end of file diff --git a/geowebservice/requirements.txt b/geowebservice/requirements.txt index 6df1f1d..7511bca 100644 --- a/geowebservice/requirements.txt +++ b/geowebservice/requirements.txt @@ -1,3 +1,3 @@ Django==1.4.5 -pysolr==3.0.6 +pysolr==3.8.1 django-cors-headers==0.13 diff --git a/geowebservice/sql/geonames_countryInfo.sql b/geowebservice/sql/geonames_countryInfo.sql deleted file mode 100644 index 7d2bbbd..0000000 --- a/geowebservice/sql/geonames_countryInfo.sql +++ /dev/null @@ -1,50 +0,0 @@ --- MySQL dump 10.13 Distrib 5.6.24, for linux-glibc2.5 (x86_64) --- --- Host: 127.0.0.1 Database: geonames --- ------------------------------------------------------ --- Server version 5.6.25-0ubuntu0.15.04.1 - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Table structure for table `countryInfo` --- - -DROP TABLE IF EXISTS `countryInfo`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `countryInfo` ( - `iso_alpha2` char(255) DEFAULT NULL, - `iso_alpha3` char(255) DEFAULT NULL, - `iso_numeric` int(11) DEFAULT NULL, - `fips_code` varchar(300) DEFAULT NULL, - `name` varchar(2000) DEFAULT NULL, - `capital` varchar(2000) DEFAULT NULL, - `areainsqkm` double DEFAULT NULL, - `population` int(11) DEFAULT NULL, - `continent` char(255) DEFAULT NULL, - `currency` char(255) DEFAULT NULL, - `currencyName` char(255) DEFAULT NULL, - `geonameId` int(11) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8; -/*!40101 SET character_set_client = @saved_cs_client */; -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - --- Dump completed on 2015-10-27 19:13:52 diff --git a/geowebservice/sql/geonames_geoname.sql b/geowebservice/sql/geonames_geoname.sql deleted file mode 100644 index c89aa64..0000000 --- a/geowebservice/sql/geonames_geoname.sql +++ /dev/null @@ -1,53 +0,0 @@ --- MySQL dump 10.13 Distrib 5.6.24, for linux-glibc2.5 (x86_64) --- --- Host: 127.0.0.1 Database: geonames --- ------------------------------------------------------ --- Server version 5.6.25-0ubuntu0.15.04.1 - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Table structure for table `geoname` --- - -DROP TABLE IF EXISTS `geoname`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `geoname` ( - `geonameid` int(11) NOT NULL, - `name` varchar(2000) CHARACTER SET utf8 DEFAULT NULL, - `asciiname` varchar(2000) CHARACTER SET utf8 DEFAULT NULL, - `alternatenames` varchar(4000) CHARACTER SET utf8 DEFAULT NULL, - `latitude` decimal(10,7) DEFAULT NULL, - `longitude` decimal(10,7) DEFAULT NULL, - `fclass` char(1) CHARACTER SET utf8 DEFAULT NULL, - `fcode` varchar(100) CHARACTER SET utf8 DEFAULT NULL, - `country` varchar(2) CHARACTER SET utf8 DEFAULT NULL, - `cc2` varchar(600) CHARACTER SET utf8 DEFAULT NULL, - `admin1` varchar(200) CHARACTER SET utf8 DEFAULT NULL, - `admin2` varchar(800) CHARACTER SET utf8 DEFAULT NULL, - `admin3` varchar(200) CHARACTER SET utf8 DEFAULT NULL, - `admin4` varchar(200) CHARACTER SET utf8 DEFAULT NULL, - UNIQUE KEY `idx_geoname_geonameid` (`geonameid`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - --- Dump completed on 2015-10-27 19:13:52 diff --git a/geowebservice/sqlite3.db b/geowebservice/sqlite3.db deleted file mode 100644 index fe1eb2f6b5ef13a93eb9bf1d70959217bd322a9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 46080 zcmeHQYiuJ&9pBkm+i`Nqz2e(ka!K~$=I*x6!;d^UL7N_F8+uLKdjKaW+dX@ebL-fy z?UXy<1I|?iQBWn2D#VAv2ZSoVP(VT;Q4kdf1QHK{DyTppK&2mm3O;}keBeK`v%9mK z%>zqrlVjZ8WcUA`-|Xz{{%7pD_|W-c!_rnNwKcP$eBeDu?TQG@Gpr>3jZ7b1OE;G8UG3Y9{z%S^pxD)W}x@W)pkdYP z#Y%Z+rCc_bo?eJYq{PyKx!$-2US^D%_4s~ZU846>Nhhrmz}?> z15DiKn3y)qQp2hl>vbqfX1!sQDyzk^QC=^F$DW>_$)@jw+kE04$Ho+IgBU8CYnD-| zHtJzf%*|z|rZeZFVBvnpLXNj!t(nD=v1VR3N>+KbaV^xy^b8oeHv&f9{!EWwLX*nR}IpGkbib@Ft zc7*`x|B7@`k}jf2)RZT8rL?_J^9qWolapw}Y?xO|7JD}Fo<%kh4BwXuyp_8 z@})6t#H?0Fv?t73{+d}klbxQ{T{U2mm%y#Lpjs+g;ILpetVVIoa#i&Q<*1rUp)0gz z^~X!_z-83oh1)9Ut-vYv)hfU|Ln0UhO}&}IF?DtpHP@-le5F>uW|;-6R)@#IT79ij zuu7ps#b*Yr6#!`8ulCQXjkziylhsIN{z*~9s znwj;S%+C1Rr|8J!wwEcduUWNXp1U;QvVitoDOL%51+J0j)9Gz4ff;x&_p$S4wb(FA zp8HJ_X3a8-<@)1~t&M1KSfZCC_^wu}>m~Zsy(uh`ODL8hDdU+9);zyntHH%|V`qtF z-!&YxvuahyCAPFfx6#G;zx~Zja0LRlG6LQ53ow!QElK(o{s#WWeyi!tN+hNZ44{up z)8UBS^w^)6e;8s^G6b>1ot`N+Jt)8!;2R{&dcIinKTB*9A*u8-JVy{QZdlD^gCh(Wei zhEok;bL!ct5I#4u&xEk2!mY!cGakCrMekC8H#pk*&DA^DF`^aw2v)AA50BL`{OL(6VbDhFuU zMaxc7;y5ikXc;4=9HnK1mWqPo2$Mp_I36d@|DAYT!hgZP!as!m{#pDn{6TEuMIefg zKtLeiA#kiyk&*-O^azO|^Qu{gY!rRV?vDdK$UxUCMA5au2ksz~yWHPWD!Jb@EPm)t_ z`tUFUkAw8-4VH939-?-GjQ^c@Lc)K=KZ0@pMf^D!)$hg=_-Xthuj1lJARrKsBvnZc zAg-UY{@*#EKu(mSbNxSlTuBb2u>RlCqo@hw_WxL~GB|(&{Xcp_8BKz=x0H)Jj&v)D z1YIIP`oGenBvg7v`#(OaBu>&(um8)36?HJ~>;E+Wk01x|f_zTCj=xI&6D_@W5ojLj zim4}0qUK{X<3aB425oQesK=l8@E9%42Kdt(d?G-*yu9?#>xYaIoPoAe9+NyrMP)l{CKpLTGp$m(5FJ4$& zKDT%grj}gLnHQa;{kO4@|J#kQurCnUy9ki?f3f~|@3xPKOd!xs1jP8?PIQHBfxzBIK#c!; zyM07-0)ciSAjbc8qAP3*1ok!pV*KCR?IWTS2(%LcGXBf*^OF2L{xay?3PvsQ2pJ_?GsUNR^OY=-~1KUoz@k& zD*_zvupq!)9H8rW9EqwI$G2Szz*U>A0^lge#PffvSESGr2<(3Z#Pk3DZ$nW7fk3MW zi1EKw-50P!+Y%^}r}ZE?HU=5n*StC?Bui$T2G6T6Ddv9VKZfhK86eKGZpanwBN1xOL5 zb*LNG+-y2y7N)$5+GY%f1Qn*og~PE4Nc5Ck6-{*94j~yHg0JN`BSSlc;OK1~g4a%B z{BQNnDf9#a`yT<(|M!0ziW&$6T17zg|5lL`dIEv{kAUd^`@aoE4Fm$MA|U#ItH=pG zfx!MpfaL!bsUk@g)Q^6I*V|m9=9&JOdT{|YmjWqIn%?wMv|;(zpoq3r8Z;>A=5m7x z(V(C|J3BF8hkALZP@N34mnse#KDp~q9b3Bz)q{&pGTc0Uf-aGL3ekBQwv7t?wKuYZ zXYhQ4#pK_0#T94s8=F_GZoBD9p6oVfkUdjEc29w&tCg#dTKNW@Y-w268_u2?j9jy- zux!@dv?cg!>$dTglDRsf)!_%yORyn~wqC9mSIbtx-J=D(z@9JVRm*@MT(6s}5Q*)I z?HWV)_V4N0dw2R)^pag=_$9dvG+u!LLw@s%HnUIP*_np7;a(Pwrrzurv=jdsdT9k) z#+b^?u4Jxe@54g zTWjv6IcX-%J3Zw`D!^?CB_3v)rQEdVG@scEsezN@qgloR-Su%xM7Q6D+QmB<6K`7A z-+EhAeJHgR@BvKNa&iyC(_;L8!){2zkw9S2ArOIx5dJrbYux^`=Ry<#2?PWJZ9!nH zkBt9A=ra<29d`fw9_;n|Y1rSliXX=Bz-O_JRgC1<FrU_Ac_$^x(_C;W7JWBIeb`BaKt7$S~l@8H5H|bHsuhlU&IaA-8S!_ z&2U3@89lV(5j$v?+D$Dh%no^!Dxr-(OuS#)$8X&-RwYYUUp3Tse z7B^4d9#e;h(Nlxot1Y>3sAY1{9r8L$=qHH#@V%?~(L(H~| ztCdQL=Mt)a>%UyuQ)gTM!JFtxu~u&wTT6mS$=pf8S~H7WULrd^<+%a_B+`btvNECt zqXdo#tAd_@H9kQWY|(2OkHRK|7GOY5$f6{^hBPeb2E{VZhXp`-CKXi|hqryDCh&0I z&F*i`jKx%4M;oX4Ob^@4lcfpR*%5y;`5&okD|BH`6Kp=1nBG8W%DVemcn`_mQ zHIc8ZafhV;WBFN0epdcDe&`mgyr|MXMBwarRDD3->IvNc$m~uevghEVADW1%XLaX? zeFDM-YZ`tEX(sC@>+G4F{R)zwX*fE*8DHj&Jh1bR*m`qpGNvYzXk*y-CIER3$0{7& z9{vshoR(hp3Ke*Pg4Y=5n-hVV+rT{IH1{f@lm6c)y-&iw#$Ul7#gE`DR^(UZm*nT< z4aoJqO+Jc#kG_nafnNu^1F}p=`mOYJ>1j~d)hE>#kroNtnD(t2$P=9Q2*hvc;X zC`b&Bx(O=B3*ntQ0z5qv$X4+#?8MZm!?v8`#isI(Y>IY>M#JZ{Tz9s_wN3kN|m+=?y6ZjGygA?%)2>c%*aB7J5(ZE~_ z*Z0^&%c((I&biZA6_WSldLEP0PJmo8#m~qPA=2-d#KZtdWadL=XgIIjM9f6LEzdl2 V9L_sOJ#!r7Q@P-LjuQnZ`Cm1NDpCLd diff --git a/geowebservice/start.sh b/geowebservice/start.sh deleted file mode 100644 index 6658c98..0000000 --- a/geowebservice/start.sh +++ /dev/null @@ -1 +0,0 @@ -docker-compose up -d --no-recreate From 7f1bbb6681d6500b9187d6aaa01f6f9794009ecc Mon Sep 17 00:00:00 2001 From: aspedrosa Date: Thu, 13 Jan 2022 20:31:59 +0000 Subject: [PATCH 2/6] migrate to django 3.2.10 (python3) --- geowebservice/.dockerignore | 3 + geowebservice/Dockerfile | 23 +-- geowebservice/django.conf | 14 -- geowebservice/docker-compose.yml | 2 +- geowebservice/geodatabase/services.py | 9 +- geowebservice/geodatabase/utils.py | 72 -------- geowebservice/geodatabase/views.py | 15 +- geowebservice/geowebservice/asgi.py | 16 ++ geowebservice/geowebservice/settings.py | 209 +++++++++++------------- geowebservice/geowebservice/urls.py | 13 +- geowebservice/geowebservice/wsgi.py | 20 +-- geowebservice/requirements.txt | 7 +- geowebservice/run.sh | 15 -- geowebservice/supervisord.conf | 8 - geowebservice/uwsgi.ini | 31 ---- geowebservice/uwsgi_params | 16 -- 16 files changed, 149 insertions(+), 324 deletions(-) create mode 100644 geowebservice/.dockerignore delete mode 100644 geowebservice/django.conf delete mode 100644 geowebservice/geodatabase/utils.py create mode 100644 geowebservice/geowebservice/asgi.py delete mode 100644 geowebservice/run.sh delete mode 100644 geowebservice/supervisord.conf delete mode 100644 geowebservice/uwsgi.ini delete mode 100644 geowebservice/uwsgi_params diff --git a/geowebservice/.dockerignore b/geowebservice/.dockerignore new file mode 100644 index 0000000..7fe462d --- /dev/null +++ b/geowebservice/.dockerignore @@ -0,0 +1,3 @@ +Dockerfile +docker-compose.yml +venv \ No newline at end of file diff --git a/geowebservice/Dockerfile b/geowebservice/Dockerfile index e74910b..5831379 100644 --- a/geowebservice/Dockerfile +++ b/geowebservice/Dockerfile @@ -1,19 +1,10 @@ -FROM debian:stretch -MAINTAINER Matt Bentley +FROM python:3.10 -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive apt install -y build-essential python python-dev python-setuptools nginx sqlite3 supervisor python-pip +COPY requirements.txt / +RUN pip install -r /requirements.txt +COPY . /app/ -ADD requirements.txt /opt/django/app/requirements.txt -RUN pip install -r /opt/django/app/requirements.txt && pip install uwsgi -ADD . /opt/django/ +WORKDIR /app -RUN (echo "daemon off;" >> /etc/nginx/nginx.conf &&\ - rm /etc/nginx/sites-enabled/default &&\ - ln -s /opt/django/django.conf /etc/nginx/sites-enabled/ &&\ - ln -s /opt/django/supervisord.conf /etc/supervisor/conf.d/) - -RUN chmod +x /opt/django/run.sh -VOLUME ["/opt/django/app"] -EXPOSE 80 -CMD ["/opt/django/run.sh"] +EXPOSE 8000 +ENTRYPOINT ["gunicorn", "-b 0.0.0.0:8000", "geowebservice.wsgi"] diff --git a/geowebservice/django.conf b/geowebservice/django.conf deleted file mode 100644 index 1cfbc12..0000000 --- a/geowebservice/django.conf +++ /dev/null @@ -1,14 +0,0 @@ -upstream django { - server unix:/opt/django/app.sock; - } - -server { - listen 80 default_server; - charset utf-8; - client_max_body_size 75M; - - location / { - uwsgi_pass django; - include /opt/django/uwsgi_params; # the uwsgi_params file you installed - } -} diff --git a/geowebservice/docker-compose.yml b/geowebservice/docker-compose.yml index f174a36..719f5a5 100644 --- a/geowebservice/docker-compose.yml +++ b/geowebservice/docker-compose.yml @@ -10,7 +10,7 @@ services: build: context: . ports: - - "8886:80" + - "8886:8000" volumes: - .:/opt/django/app depends_on: diff --git a/geowebservice/geodatabase/services.py b/geowebservice/geodatabase/services.py index db8e6da..5ff3d72 100644 --- a/geowebservice/geodatabase/services.py +++ b/geowebservice/geodatabase/services.py @@ -16,15 +16,12 @@ """A module to index .. moduleauthor:: Luís A. Bastião Silva """ - -from __future__ import print_function +import csv import logging import pysolr from django.conf import settings -from geodatabase.utils import UnicodeReader - logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG) @@ -53,7 +50,7 @@ def __init__(self): def load_country_info(self, countryFile): with open(countryFile, "rU") as ifile: - reader = UnicodeReader(ifile, delimiter='\t', ) + reader = csv.reader(ifile, delimiter="\t") self.countryInfo = {row[11]: {'continent': row[8], 'name': row[4]} for row in reader} @@ -61,7 +58,7 @@ def load_initial_data(self, allCountriesFile): logger.info("Load initial data to Solr") with open(allCountriesFile) as csvfile: - spamreader = UnicodeReader(csvfile, delimiter='\t', quotechar='|') + spamreader = csv.reader(csvfile, delimiter='\t', quotechar='|') list_docs_to_commit = [] i = 0 diff --git a/geowebservice/geodatabase/utils.py b/geowebservice/geodatabase/utils.py deleted file mode 100644 index 696a091..0000000 --- a/geowebservice/geodatabase/utils.py +++ /dev/null @@ -1,72 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (C) 2015 Universidade de Aveiro, DETI/IEETA, Bioinformatics Group - http://bioinformatics.ua.pt/ -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# From http://stackoverflow.com/questions/17245415/read-and-write-csv-files-including-unicode-with-python-2-7 -import cStringIO -import codecs -import csv - - -class UTF8Recoder: - def __init__(self, f, encoding): - self.reader = codecs.getreader(encoding)(f) - - def __iter__(self): - return self - - def next(self): - return self.reader.next().encode("utf-8") - - -class UnicodeReader: - def __init__(self, f, dialect=csv.excel, encoding="utf-8-sig", **kwds): - f = UTF8Recoder(f, encoding) - self.reader = csv.reader(f, dialect=dialect, **kwds) - - def next(self): - """ - next() -> unicode - This function reads and returns the next line as a Unicode string. - """ - row = self.reader.next() - return [unicode(s, "utf-8") for s in row] - - def __iter__(self): - return self - - -class UnicodeWriter: - def __init__(self, f, dialect=csv.excel, encoding="utf-8-sig", **kwds): - self.queue = cStringIO.StringIO() - self.writer = csv.writer(self.queue, dialect=dialect, **kwds) - self.stream = f - self.encoder = codecs.getincrementalencoder(encoding)() - - def writerow(self, row): - """ - writerow(unicode) -> None - This function takes a Unicode string and encodes it to the output. - """ - self.writer.writerow([s.encode("utf-8") for s in row]) - data = self.queue.getvalue() - data = data.decode("utf-8") - data = self.encoder.encode(data) - self.stream.write(data) - self.queue.truncate(0) - - def writerows(self, rows): - for row in rows: - self.writerow(row) diff --git a/geowebservice/geodatabase/views.py b/geowebservice/geodatabase/views.py index b24427c..0e5d1f4 100644 --- a/geowebservice/geodatabase/views.py +++ b/geowebservice/geodatabase/views.py @@ -144,20 +144,15 @@ def detail(request, geonameid): def buildJson(response_object, response_data): """ Builds the answer with multiples geonames entries""" for i in range(0, len(response_object) - 1): - print(response_object[i]) - response_data = response_data + addEntry(response_object[i], response_data) + response_data = response_data + addEntry(response_object[i]) return response_data -def addEntry(geoname, response_data): +def addEntry(geoname): """ Handles a GeoName entry """ - response = {} - print(geoname) - response['geonameid'] = int(geoname['geonameId_t']) - response['name'] = geoname['name_t'] - response['fcode'] = geoname['fcode_t'] + response = {'geonameid': int(geoname['geonameId_t']), 'name': geoname['name_t'], 'fcode': geoname['fcode_t']} try: response['country'] = geoname['country_t'] response['adm1'] = geoname['admin1_t'] @@ -188,13 +183,13 @@ def getCoordinates(request, location): if fcode == 'ADM1' and response_object.hits == 0: response_object = solr.search("fcode_t:ISLS AND name_t:" + name) response_object = response_object.docs[0] - response_data = buildCoordinates(response_object, []) + response_data = buildCoordinates(response_object) return HttpResponse(json.dumps(response_data), content_type="application/json") except: return HttpResponse(json.dumps([]), content_type="application/json") -def buildCoordinates(response_object, response_data): +def buildCoordinates(response_object): """ Build a list with latitude and longitude of the location """ diff --git a/geowebservice/geowebservice/asgi.py b/geowebservice/geowebservice/asgi.py new file mode 100644 index 0000000..ceb3cca --- /dev/null +++ b/geowebservice/geowebservice/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for geowebservice project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'geowebservice.settings') + +application = get_asgi_application() diff --git a/geowebservice/geowebservice/settings.py b/geowebservice/geowebservice/settings.py index aad1363..ef08264 100644 --- a/geowebservice/geowebservice/settings.py +++ b/geowebservice/geowebservice/settings.py @@ -1,124 +1,111 @@ # Django settings for geowebservice project. + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / "subdir". +BASE_DIR = Path(__file__).resolve().parent.parent + +# Make this unique, and don"t share it with anybody. +SECRET_KEY = "%suk*!m42-@mw+)))6%g6u=dpwseyxew&zij619qovu&=#4yk#" + DEBUG = True -TEMPLATE_DEBUG = DEBUG -ADMINS = ( - # ('Your Name', 'your_email@example.com'), +ALLOWED_HOSTS = ["*"] + +INSTALLED_APPS = ( + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + + "corsheaders", + "geodatabase", ) -MANAGERS = ADMINS +MIDDLEWARE = ( + "corsheaders.middleware.CorsMiddleware", + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +) + +ROOT_URLCONF = "geowebservice.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "geowebservice.wsgi.application" +# Database +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases DATABASES = { - 'default' : { - 'ENGINE' :'django.db.backends.sqlite3', - 'NAME' : 'sqlite3.db', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", } } +# Password validation +# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] -# Hosts/domain names that are valid for this site; required if DEBUG is False -# See https://docs.djangoproject.com/en/1.4/ref/settings/#allowed-hosts -ALLOWED_HOSTS = ["*"] - -# Local time zone for this installation. Choices can be found here: -# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name -# although not all choices may be available on all operating systems. -# In a Windows environment this must be set to your system time zone. -TIME_ZONE = 'America/Chicago' - -# Language code for this installation. All choices can be found here: -# http://www.i18nguy.com/unicode/language-identifiers.html -LANGUAGE_CODE = 'en-us' +# Internationalization +# https://docs.djangoproject.com/en/3.2/topics/i18n/ +LANGUAGE_CODE = "en-us" -SITE_ID = 1 +TIME_ZONE = "America/Chicago" -# If you set this to False, Django will make some optimizations so as not -# to load the internationalization machinery. USE_I18N = True -# If you set this to False, Django will not format dates, numbers and -# calendars according to the current locale. USE_L10N = True -# If you set this to False, Django will not use timezone-aware datetimes. USE_TZ = True -# Absolute filesystem path to the directory that will hold user-uploaded files. -# Example: "/home/media/media.lawrence.com/media/" -MEDIA_ROOT = '' - -# URL that handles the media served from MEDIA_ROOT. Make sure to use a -# trailing slash. -# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" -MEDIA_URL = '' - -# Absolute path to the directory static files should be collected to. -# Don't put anything in this directory yourself; store your static files -# in apps' "static/" subdirectories and in STATICFILES_DIRS. -# Example: "/home/media/media.lawrence.com/static/" -STATIC_ROOT = '' - -# URL prefix for static files. -# Example: "http://media.lawrence.com/static/" -STATIC_URL = '/static/' - -# Additional locations of static files -STATICFILES_DIRS = ( - # Put strings here, like "/home/html/static" or "C:/www/django/static". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. -) - -# List of finder classes that know how to find static files in -# various locations. -STATICFILES_FINDERS = ( - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', -# 'django.contrib.staticfiles.finders.DefaultStorageFinder', -) +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.2/howto/static-files/ +STATIC_URL = "/static/" -# Make this unique, and don't share it with anybody. -SECRET_KEY = '%suk*!m42-@mw+)))6%g6u=dpwseyxew&zij619qovu&=#4yk#' +# Default primary key field type +# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" -# List of callables that know how to import templates from various sources. -TEMPLATE_LOADERS = ( - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', -) -MIDDLEWARE_CLASSES = ( - 'corsheaders.middleware.CorsMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - # Uncomment the next line for simple clickjacking protection: - # 'django.middleware.clickjacking.XFrameOptionsMiddleware', +ADMINS = ( + # ("Your Name", "your_email@example.com"), ) -ROOT_URLCONF = 'geowebservice.urls' - -# Python dotted path to the WSGI application used by Django's runserver. -WSGI_APPLICATION = 'geowebservice.wsgi.application' - -TEMPLATE_DIRS = ( - # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. -) +MANAGERS = ADMINS -INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.messages', - 'django.contrib.staticfiles', - - 'corsheaders', - 'geodatabase', -) # A sample logging configuration. The only tangible logging # performed by this configuration is to send an email to @@ -126,25 +113,25 @@ # See http://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse' + "version": 1, + "disable_existing_loggers": False, + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse" } }, - 'handlers': { - 'mail_admins': { - 'level': 'ERROR', - 'filters': ['require_debug_false'], - 'class': 'django.utils.log.AdminEmailHandler' + "handlers": { + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false"], + "class": "django.utils.log.AdminEmailHandler" } }, - 'loggers': { - 'django.request': { - 'handlers': ['mail_admins'], - 'level': 'ERROR', - 'propagate': True, + "loggers": { + "django.request": { + "handlers": ["mail_admins"], + "level": "ERROR", + "propagate": True, }, } } diff --git a/geowebservice/geowebservice/urls.py b/geowebservice/geowebservice/urls.py index 2ad0bd9..d6d5c70 100644 --- a/geowebservice/geowebservice/urls.py +++ b/geowebservice/geowebservice/urls.py @@ -1,11 +1,10 @@ -from django.conf.urls import patterns, url - -urlpatterns = patterns( - "", +from django.urls import re_path +from geodatabase import views +urlpatterns = [ # get child locations by geonameid - url(r'^geodatabase/(?P\d+)/$', 'geodatabase.views.detail'), + re_path(r"^geodatabase/(?P\d+)/$", views.detail), # get coordinates by name and fcode - url(r'^geodatabase/(?P[a-zA-Z, ]+)/$', 'geodatabase.views.getCoordinates'), -) + re_path(r"^geodatabase/(?P[a-zA-Z, ]+)/$", views.getCoordinates), +] diff --git a/geowebservice/geowebservice/wsgi.py b/geowebservice/geowebservice/wsgi.py index a924bfb..3da33e6 100644 --- a/geowebservice/geowebservice/wsgi.py +++ b/geowebservice/geowebservice/wsgi.py @@ -1,24 +1,16 @@ """ WSGI config for geowebservice project. -This module contains the WSGI application used by Django's development server -and any production WSGI deployments. It should expose a module-level variable -named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover -this application via the ``WSGI_APPLICATION`` setting. - -Usually you will have the standard Django WSGI application here, but it also -might make sense to replace the whole Django WSGI application with a custom one -that later delegates to the Django one. For example, you could introduce WSGI -middleware here, or combine a Django application with an application of another -framework. +It exposes the WSGI callable as a module-level variable named ``application``. +For more information on this file, see +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ """ + import os + from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "geowebservice.settings") +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'geowebservice.settings') -# This application object is used by any WSGI server configured to use this -# file. This includes Django's development server, if the WSGI_APPLICATION -# setting points here. application = get_wsgi_application() diff --git a/geowebservice/requirements.txt b/geowebservice/requirements.txt index 7511bca..e800a3c 100644 --- a/geowebservice/requirements.txt +++ b/geowebservice/requirements.txt @@ -1,3 +1,4 @@ -Django==1.4.5 -pysolr==3.8.1 -django-cors-headers==0.13 +Django==3.2.11 +django-cors-headers==3.11.0 +gunicorn==20.1.0 +pysolr==3.8.1 \ No newline at end of file diff --git a/geowebservice/run.sh b/geowebservice/run.sh deleted file mode 100644 index 0ccb684..0000000 --- a/geowebservice/run.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -e - -MODULE=${MODULE:-website} - -sed -i "s#module=website.wsgi:application#module=${MODULE}.wsgi:application#g" /opt/django/uwsgi.ini - -if [ ! -f "/opt/django/app/manage.py" ] -then - echo "creating basic django project (module: ${MODULE})" - django-admin.py startproject ${MODULE} /opt/django/app/ -fi - -exec /usr/bin/supervisord diff --git a/geowebservice/supervisord.conf b/geowebservice/supervisord.conf deleted file mode 100644 index 2a8d336..0000000 --- a/geowebservice/supervisord.conf +++ /dev/null @@ -1,8 +0,0 @@ -[supervisord] -nodaemon=true - -[program:uwsgi] -command = /usr/local/bin/uwsgi --ini /opt/django/uwsgi.ini --touch-reload /opt/django/reload - -[program:nginx] -command = /usr/sbin/nginx diff --git a/geowebservice/uwsgi.ini b/geowebservice/uwsgi.ini deleted file mode 100644 index a1f0c4b..0000000 --- a/geowebservice/uwsgi.ini +++ /dev/null @@ -1,31 +0,0 @@ -[uwsgi] -# this config will be loaded if nothing specific is specified -# load base config from below -ini = :base - -# %d is the dir this configuration file is in -socket = %dapp.sock -master = true -processes = 4 - -[dev] -ini = :base -# socket (uwsgi) is not the same as http, nor http-socket -socket = :8001 - - -[local] -ini = :base -http = :8000 -# set the virtual env to use -home=/Users/you/envs/env - - -[base] -# chdir to the folder of this config file, plus app/website -chdir = /opt/django -# load the module from wsgi.py, it is a python path from -# the directory above. -module=geowebservice.wsgi:application -# allow anyone to connect to the socket. This is very permissive -chmod-socket=666 diff --git a/geowebservice/uwsgi_params b/geowebservice/uwsgi_params deleted file mode 100644 index f539451..0000000 --- a/geowebservice/uwsgi_params +++ /dev/null @@ -1,16 +0,0 @@ - -uwsgi_param QUERY_STRING $query_string; -uwsgi_param REQUEST_METHOD $request_method; -uwsgi_param CONTENT_TYPE $content_type; -uwsgi_param CONTENT_LENGTH $content_length; - -uwsgi_param REQUEST_URI $request_uri; -uwsgi_param PATH_INFO $document_uri; -uwsgi_param DOCUMENT_ROOT $document_root; -uwsgi_param SERVER_PROTOCOL $server_protocol; -uwsgi_param HTTPS $https if_not_empty; - -uwsgi_param REMOTE_ADDR $remote_addr; -uwsgi_param REMOTE_PORT $remote_port; -uwsgi_param SERVER_PORT $server_port; -uwsgi_param SERVER_NAME $server_name; From 119d4c724d00a8e66d0d348587fd7a1b8c9a99be Mon Sep 17 00:00:00 2001 From: aspedrosa Date: Fri, 14 Jan 2022 00:00:28 +0000 Subject: [PATCH 3/6] improve deployment --- geowebservice/Dockerfile | 10 ++++++---- geowebservice/docker-compose.yml | 4 ---- .../management/commands/load_data.py | 8 ++++---- geowebservice/geodatabase/services.py | 12 ++++-------- geowebservice/geodatabase/views.py | 3 +-- geowebservice/geowebservice/settings.py | 9 +++++---- geowebservice/start.sh | 19 +++++++++++++++++++ 7 files changed, 39 insertions(+), 26 deletions(-) create mode 100755 geowebservice/start.sh diff --git a/geowebservice/Dockerfile b/geowebservice/Dockerfile index 5831379..06de7a0 100644 --- a/geowebservice/Dockerfile +++ b/geowebservice/Dockerfile @@ -1,10 +1,12 @@ FROM python:3.10 -COPY requirements.txt / -RUN pip install -r /requirements.txt -COPY . /app/ - WORKDIR /app +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY . . + EXPOSE 8000 + ENTRYPOINT ["gunicorn", "-b 0.0.0.0:8000", "geowebservice.wsgi"] diff --git a/geowebservice/docker-compose.yml b/geowebservice/docker-compose.yml index 719f5a5..3eec830 100644 --- a/geowebservice/docker-compose.yml +++ b/geowebservice/docker-compose.yml @@ -3,15 +3,11 @@ version: "3" services: solr: image: solr:5.5.3 - ports: - - 8983:8983 geodropdownservice: build: context: . ports: - "8886:8000" - volumes: - - .:/opt/django/app depends_on: - solr diff --git a/geowebservice/geodatabase/management/commands/load_data.py b/geowebservice/geodatabase/management/commands/load_data.py index 97c300d..1555099 100644 --- a/geowebservice/geodatabase/management/commands/load_data.py +++ b/geowebservice/geodatabase/management/commands/load_data.py @@ -13,12 +13,12 @@ class Command(BaseCommand): help = 'Closes the specified poll for voting' def add_arguments(self, parser): - parser.add_argument('all', nargs='+', type=str) - parser.add_argument('countries', nargs='+', type=str) + parser.add_argument('countries', type=str) + parser.add_argument('all', type=str) def handle(self, *args, **options): - all_locations = options['all'][0] - countries_names = options['countries'][0] + countries_names = options['countries'] + all_locations = options['all'] s = ServiceSolr() s.load_country_info(countries_names) diff --git a/geowebservice/geodatabase/services.py b/geowebservice/geodatabase/services.py index 5ff3d72..738d354 100644 --- a/geowebservice/geodatabase/services.py +++ b/geowebservice/geodatabase/services.py @@ -39,16 +39,14 @@ def __init__(self): logger.info("Initial Solr Service") self.solr = pysolr.Solr( - 'http://{}:{}/solr/{}/'.format( - settings.SOLR_HOST, - settings.SOLR_PORT, - settings.SOLR_CORE, - ), + f"http://{settings.SOLR_HOST}:{settings.SOLR_PORT}/solr/{settings.SOLR_CORE}/", timeout=self.CONNECTION_TIMEOUT_DEFAULT, ) logger.info("Connected to Solr") def load_country_info(self, countryFile): + logger.info("Loading country info") + with open(countryFile, "rU") as ifile: reader = csv.reader(ifile, delimiter="\t") @@ -63,8 +61,7 @@ def load_initial_data(self, allCountriesFile): i = 0 for row in spamreader: - if not (row[1] == 'Earth' or row[7] == 'CONT' or row[7] == 'PCLI' or row[7] == 'ISLS' or - row[7] == 'ADM1' or row[7] == 'ADM2' or row[7] == 'ADM3' or row[7] == 'ADM4'): + if not (row[1] == 'Earth' or row[7] == 'CONT' or row[7] == 'PCLI' or row[7] == 'ISLS' or row[7] == 'ADM1' or row[7] == 'ADM2' or row[7] == 'ADM3' or row[7] == 'ADM4'): continue i = i + 1 d = {'id': i, 'geonameId_t': row[0], 'name_t': row[1], 'asciiname_t': row[2], @@ -78,7 +75,6 @@ def load_initial_data(self, allCountriesFile): d['continent_t'] = self.countryInfo[row[0]]['continent'] d['name_t'] = self.countryInfo[row[0]]['name'] - d = dict(d.items()) list_docs_to_commit.append(d) if i % 60000 == 0: self.solr.add(list_docs_to_commit) diff --git a/geowebservice/geodatabase/views.py b/geowebservice/geodatabase/views.py index 0e5d1f4..9c371b5 100644 --- a/geowebservice/geodatabase/views.py +++ b/geowebservice/geodatabase/views.py @@ -36,10 +36,10 @@ def detail(request, geonameid): it will analyse what type of location that it handles and then returns the proper locations according to the hierarchy. """ + solr = ServiceSolr() # TODO: validate geonameid properly. if geonameid != "0": - solr = ServiceSolr() results = solr.geonameId(geonameid) # Check it, if no results in the geonames database, it returns an error @@ -87,7 +87,6 @@ def detail(request, geonameid): response_data = buildJson(response_object.docs, response_data) elif name == 'Mundus': - solr = ServiceSolr() response_object = solr.search( "continent_t:EU OR continent_t:AF OR continent_t:OC OR continent_t:SA OR continent_t:NA OR continent_t:AS") response_data = buildJson(response_object.docs, response_data) diff --git a/geowebservice/geowebservice/settings.py b/geowebservice/geowebservice/settings.py index ef08264..6fc6310 100644 --- a/geowebservice/geowebservice/settings.py +++ b/geowebservice/geowebservice/settings.py @@ -1,5 +1,5 @@ # Django settings for geowebservice project. - +import os from pathlib import Path # Build paths inside the project like this: BASE_DIR / "subdir". @@ -136,8 +136,9 @@ } } -SOLR_HOST = "solr" -SOLR_PORT = "8983" -SOLR_CORE = "geonames" + +SOLR_HOST = os.environ.get("SOLR_HOST", "solr") +SOLR_PORT = os.environ.get("SOLR_PORT", "8983") +SOLR_CORE = os.environ.get("SOLR_CORE", "geonames") CORS_ORIGIN_ALLOW_ALL = True diff --git a/geowebservice/start.sh b/geowebservice/start.sh new file mode 100755 index 0000000..8b1c20c --- /dev/null +++ b/geowebservice/start.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +docker-compose up -d solr + +until docker-compose exec -T solr sh -c "solr status" | grep running ; do + sleep 2 +done + +docker-compose exec -T solr sh -c "solr create_core -c geonames -d basic_configs" + +docker-compose up -d geodropdownservice + +docker-compose exec -T geodropdownservice sh -c """ +set -x +wget -P /tmp http://download.geonames.org/export/dump/allCountries.zip +unzip /tmp/allCountries.zip -d /tmp +python manage.py load_data country.csv /tmp/allCountries.txt +rm /tmp/allCountries.* +""" From 9759a4249c4d579cf40fcf87beea23bebb21078e Mon Sep 17 00:00:00 2001 From: aspedrosa Date: Fri, 14 Jan 2022 12:57:07 +0000 Subject: [PATCH 4/6] improve readme instruction on demo --- README.md | 37 ++++++++++++++++++++++++-------- geowebservice/docker-compose.yml | 5 +++++ geowebservice/start.sh | 27 +++++++++++++++-------- package.json | 11 ---------- 4 files changed, 51 insertions(+), 29 deletions(-) delete mode 100644 package.json diff --git a/README.md b/README.md index 6895463..1040f28 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # GeoDropdown GeoDropdown makes use of geowebservice to create a multiple drodpdown lists system, which guarantees that users will provides well formatted locations, according to ISO-3166. -Live demo: [demo.html](demo.html) - # Hierarchical levels There are 7 hierarchical levels, which corresponds to 7 dropdown list: @@ -73,17 +71,34 @@ Portugal (wich is a country) geonameid is 2264397, so you can use the link /geod [{"geonameid": 2262961, "fcode": "ADM1", "name": "Distrito de Set\u00c3\u00babal", "adm1": 19}, {"geonameid": 2263478, "fcode": "ADM1", "name": "Distrito de Santar\u00c3\u00a9m", "adm1": 18}, {"geonameid": 2264507, "fcode": "ADM1", "name": "Distrito de Portalegre", "adm1": 16}, {"geonameid": 2267056, "fcode": "ADM1", "name": "Distrito de Lisboa", "adm1": 14}, {"geonameid": 2267094, "fcode": "ADM1", "name": "Distrito de Leiria", "adm1": 13}, {"geonameid": 2268337, "fcode": "ADM1", "name": "Distrito de Faro", "adm1": 9}, {"geonameid": 2268404, "fcode": "ADM1", "name": "Distrito de \u00c3\u2030vora", "adm1": 8}, {"geonameid": 2269513, "fcode": "ADM1", "name": "Distrito de Castelo Branco", "adm1": 6}, {"geonameid": 2270984, "fcode": "ADM1", "name": "Distrito de Beja", "adm1": 3}, {"geonameid": 2593105, "fcode": "ADM1", "name": "Madeira", "adm1": 10}, {"geonameid": 2732264, "fcode": "ADM1", "name": "Distrito de Viseu", "adm1": 22}, {"geonameid": 2732437, "fcode": "ADM1", "name": "Distrito de Vila Real", "adm1": 21}, {"geonameid": 2732772, "fcode": "ADM1", "name": "Distrito de Viana do Castelo", "adm1": 20}, {"geonameid": 2735941, "fcode": "ADM1", "name": "Distrito do Porto", "adm1": 17}, {"geonameid": 2738782, "fcode": "ADM1", "name": "Distrito da Guarda", "adm1": 11}, {"geonameid": 2740636, "fcode": "ADM1", "name": "Distrito de Coimbra", "adm1": 7}, {"geonameid": 2742026, "fcode": "ADM1", "name": "Distrito de Bragan\u00c3\u00a7a", "adm1": 5}, {"geonameid": 2742031, "fcode": "ADM1", "name": "Distrito de Braga", "adm1": 4}, {"geonameid": 2742610, "fcode": "ADM1", "name": "Distrito de Aveiro", "adm1": 2}] ``` -# Server Side: -# How to make it run -There is a docker file to run this application. Therefore, if you want to run geodropdown, just go to geowebservice folder and run the following commands: +# Setup demo +## Front end: +```sh +npm install -g grunt bower serve + +# download external dependencies +bower install + +# build dist files +grunt + +# start a dev server to test demo.html +serve +``` + +## Server Side: +### How to make it run +There is a docker-compose file to run this application. Therefore, if you want to run geodropdown, just go to geowebservice folder and run the following commands: ``` -sh build.sh -docker-compose up +cd geowebservice +docker-compose build +./start.sh ``` -Now, you need to have data related to all locations around the world +### How to load data to the Web Service databases + +**Note:** Data is automatically loaded with the start.sh script. You can ignore this section. -# How to load data to the Web Service databases To load all the required data, you will need two files from geonames: allCountries.txt and countryInfo.txt. You can find them here: [files link](http://download.geonames.org/export/dump/) @@ -91,6 +106,9 @@ The next step is to give the location of those files to the webservice. So you n Now you are prepared to load all locations, simply execute python services.py in console. +## Test it +Go to [http://localhost:3000/demo](http://localhost:3000/demo) + # Authors: - Leonardo Coelho - @@ -98,6 +116,7 @@ Now you are prepared to load all locations, simply execute python services.py in # Contributors - Renato Pinho - Luis A. Bastião Silva - +- André Pedrosa - # Mantainers diff --git a/geowebservice/docker-compose.yml b/geowebservice/docker-compose.yml index 3eec830..229a65b 100644 --- a/geowebservice/docker-compose.yml +++ b/geowebservice/docker-compose.yml @@ -3,6 +3,8 @@ version: "3" services: solr: image: solr:5.5.3 + volumes: + - geodata:/opt/solr/server/solr geodropdownservice: build: @@ -11,3 +13,6 @@ services: - "8886:8000" depends_on: - solr + +volumes: + geodata: diff --git a/geowebservice/start.sh b/geowebservice/start.sh index 8b1c20c..641cb65 100755 --- a/geowebservice/start.sh +++ b/geowebservice/start.sh @@ -1,19 +1,28 @@ #!/bin/sh +PREVIOUS_PWD=$(pwd) +cd $(dirname "$0") + docker-compose up -d solr until docker-compose exec -T solr sh -c "solr status" | grep running ; do sleep 2 done -docker-compose exec -T solr sh -c "solr create_core -c geonames -d basic_configs" +( + set -e + + docker-compose exec -T solr sh -c "solr create_core -c geonames -d basic_configs" + + docker-compose up -d geodropdownservice -docker-compose up -d geodropdownservice + docker-compose exec -T geodropdownservice sh -c """ + set -x + wget -P /tmp http://download.geonames.org/export/dump/allCountries.zip + unzip /tmp/allCountries.zip -d /tmp + python manage.py load_data country.csv /tmp/allCountries.txt + rm /tmp/allCountries.* + """ +) -docker-compose exec -T geodropdownservice sh -c """ -set -x -wget -P /tmp http://download.geonames.org/export/dump/allCountries.zip -unzip /tmp/allCountries.zip -d /tmp -python manage.py load_data country.csv /tmp/allCountries.txt -rm /tmp/allCountries.* -""" +cd $PREVIOUS_PWD diff --git a/package.json b/package.json deleted file mode 100644 index ea216ff..0000000 --- a/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "GeoDropdown", - "version": "0.0.0", - "description": "dropdown lists location system", - "devDependencies": { - "grunt": "^0.4.5", - "grunt-contrib-copy": "^0.8.1", - "grunt-contrib-htmlmin": "^0.5.0", - "grunt-contrib-uglify": "^0.9.2" - } -} From 0d37bbd73ae8bf14ea75af6d4f4bb55ac4a1acc1 Mon Sep 17 00:00:00 2001 From: aspedrosa Date: Fri, 14 Jan 2022 15:10:05 +0000 Subject: [PATCH 5/6] improve gitignore --- .gitignore | 154 +++++++++++++++++++++++++++++++++++- geowebservice/.dockerignore | 3 +- 2 files changed, 155 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 4c43438..a88987d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,156 @@ bower_components/ node_modules/ + +# Retrieved from https://github.com/github/gitignore/blob/752b7678db497edb377fa982ded0c70035393794/Python.gitignore +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ dist/ -*.pyc +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/geowebservice/.dockerignore b/geowebservice/.dockerignore index 7fe462d..bb2655f 100644 --- a/geowebservice/.dockerignore +++ b/geowebservice/.dockerignore @@ -1,3 +1,4 @@ Dockerfile docker-compose.yml -venv \ No newline at end of file +venv +db.sqlite3 From 1d58e7b49938838c935653ed903088b3bcad47a0 Mon Sep 17 00:00:00 2001 From: aspedrosa Date: Fri, 14 Jan 2022 15:10:36 +0000 Subject: [PATCH 6/6] fix urls --- geowebservice/geowebservice/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geowebservice/geowebservice/urls.py b/geowebservice/geowebservice/urls.py index d6d5c70..bba6a23 100644 --- a/geowebservice/geowebservice/urls.py +++ b/geowebservice/geowebservice/urls.py @@ -3,8 +3,8 @@ urlpatterns = [ # get child locations by geonameid - re_path(r"^geodatabase/(?P\d+)/$", views.detail), + re_path(r"^geodropdown/geodatabase/(?P\d+)/$", views.detail), # get coordinates by name and fcode - re_path(r"^geodatabase/(?P[a-zA-Z, ]+)/$", views.getCoordinates), + re_path(r"^geodropdown/geodatabase/(?P[a-zA-Z, ]+)/$", views.getCoordinates), ]