Skip to content

Commit

Permalink
Merge branch 'master' into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
@olimpiurob
olimpiurob authored Feb 14, 2022
2 parents d70233d + 5821289 commit 0994450
Show file tree
Hide file tree
Showing 52 changed files with 934 additions and 277 deletions.
25 changes: 14 additions & 11 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ pipeline {

environment {
GIT_NAME = "Products.Reportek"
GIT_SRC = "https://github.com/eea/Products.Reportek.git"
SONARQUBE_TAGS = "bdr.eionet.europa.eu,cdr.eionet.europa.eu,mdr.eionet.europa.eu"
}

stages {

stages {
stage('Cosmetics') {
steps {
parallel(
Expand All @@ -16,7 +17,7 @@ pipeline {
node(label: 'docker') {
script {
catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
sh '''docker run -i --rm --name="$BUILD_TAG-jshint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jshint'''
sh '''docker run -i --rm --name="$BUILD_TAG-jshint" -e GIT_SRC="$GIT_SRC" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jshint'''
}
}
}
Expand All @@ -26,7 +27,7 @@ pipeline {
node(label: 'docker') {
script {
catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
sh '''docker run -i --rm --name="$BUILD_TAG-csslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/csslint'''
sh '''docker run -i --rm --name="$BUILD_TAG-csslint" -e GIT_SRC="$GIT_SRC" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/csslint'''
}
}
}
Expand All @@ -36,7 +37,7 @@ pipeline {
node(label: 'docker') {
script {
catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
sh '''docker run -i --rm --name="$BUILD_TAG-pep8" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pep8'''
sh '''docker run -i --rm --name="$BUILD_TAG-pep8" -e GIT_SRC="$GIT_SRC" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pep8'''
}
}
}
Expand All @@ -46,7 +47,7 @@ pipeline {
node(label: 'docker') {
script {
catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
sh '''docker run -i --rm --name="$BUILD_TAG-pylint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pylint'''
sh '''docker run -i --rm --name="$BUILD_TAG-pylint" -e GIT_SRC="$GIT_SRC" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pylint'''
}
}
}
Expand All @@ -68,25 +69,26 @@ pipeline {

"JS Lint": {
node(label: 'docker') {
sh '''docker run -i --rm --name="$BUILD_TAG-jslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jslint4java'''
sh '''docker run -i --rm --name="$BUILD_TAG-jslint" -e GIT_SRC="$GIT_SRC" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jslint4java'''
}
},

"Flake8": {
node(label: 'docker') {
sh '''docker run -i --rm --pull=always --name="$BUILD_TAG-flake8" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/flake8 flake8 --extend-ignore=W605,W606'''
sh '''docker run -i --rm --pull=always --name="$BUILD_TAG-flake8" -e GIT_SRC="$GIT_SRC" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/flake8 flake8 --extend-ignore=W605,W606'''
}
},

"i18n": {
node(label: 'docker') {
sh '''docker run -i --rm --name=$BUILD_TAG-i18n -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/i18ndude'''
sh '''docker run -i --rm --name=$BUILD_TAG-i18n -e GIT_SRC="$GIT_SRC" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/i18ndude'''
}
}
)
}
}


stage('Report to SonarQube') {
steps {
node(label: 'docker') {
Expand Down Expand Up @@ -116,7 +118,6 @@ pipeline {
}



stage('Pull Request') {
when {
not {
Expand Down Expand Up @@ -148,7 +149,7 @@ pipeline {
steps {
node(label: 'docker') {
withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'eea-jenkins', usernameVariable: 'EGGREPO_USERNAME', passwordVariable: 'EGGREPO_PASSWORD'],string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN'),[$class: 'UsernamePasswordMultiBinding', credentialsId: 'pypi-jenkins', usernameVariable: 'PYPI_USERNAME', passwordVariable: 'PYPI_PASSWORD']]) {
sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-master" -e GIT_BRANCH="$BRANCH_NAME" -e EGGREPO_USERNAME="$EGGREPO_USERNAME" -e EGGREPO_PASSWORD="$EGGREPO_PASSWORD" -e GIT_NAME="$GIT_NAME" -e PYPI_USERNAME="$PYPI_USERNAME" -e PYPI_PASSWORD="$PYPI_PASSWORD" -e GIT_ORG="$GIT_ORG" -e GIT_TOKEN="$GITHUB_TOKEN" eeacms/gitflow'''
sh '''docker pull eeacms/gitflow; docker run -i --rm --name="$BUILD_TAG-gitflow-master" -e GIT_BRANCH="$BRANCH_NAME" -e EGGREPO_USERNAME="$EGGREPO_USERNAME" -e EGGREPO_PASSWORD="$EGGREPO_PASSWORD" -e GIT_NAME="$GIT_NAME" -e PYPI_USERNAME="$PYPI_USERNAME" -e PYPI_PASSWORD="$PYPI_PASSWORD" -e GIT_ORG="$GIT_ORG" -e GIT_TOKEN="$GITHUB_TOKEN" eeacms/gitflow'''
}
}
}
Expand Down Expand Up @@ -179,3 +180,5 @@ pipeline {
}
}
}


54 changes: 35 additions & 19 deletions Products/Reportek/BaseRemoteApplication.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ def add_zip_feedback(self, archive, fb, files, wk, l_file_id, l_ret,
feedback_ob.content_type = 'text/html'
fb_status, fb_message = self.extract_metadata(feedback_ob)
feedback_ob.feedback_status = fb_status
if fb_status == 'BLOCKER':
wk.blocker = True
feedback_ob.message = fb_message
else:
feedback_ob.manage_uploadFeedback(archive, filename=f_name)
Expand All @@ -56,22 +58,36 @@ def handle_remote_file(self, url, l_file_id, workitem_id, l_ret, job_id,
restricted=False):
""""""
wk = getattr(self, workitem_id)
r = requests.get(url, allow_redirects=True,
headers={'Authorization': self.token},
verify=False)
from contextlib import closing
zip = io.BytesIO(r.content)
with closing(r), zip_content.ZZipFile(zip) as archive:
fbs = {}
for name in archive.namelist():
k = name.split('/')[-1].split('.')[0]
if k:
if k not in fbs.keys():
fbs[k] = [name]
else:
fbs[k].append(name)
for fb in fbs:
self.add_zip_feedback(archive, fb, fbs[fb], wk,
l_file_id, l_ret, job_id,
restricted=restricted)
transaction.commit()
result = {}
try:
r = requests.get(url, allow_redirects=True,
headers={'Authorization': self.token},
verify=False)
result['status_code'] = r.status_code
result['url'] = url
result['content_type'] = r.headers.get('Content-Type', '')
result['content_lenght'] = len(r.content)
if r.status_code == requests.codes.ok:
from contextlib import closing
zip = io.BytesIO(r.content)
with closing(r), zip_content.ZZipFile(zip) as archive:
fbs = {}
for name in archive.namelist():
k = name.split('/')[-1].split('.')[0]
if k:
if k not in fbs.keys():
fbs[k] = [name]
else:
fbs[k].append(name)
for fb in fbs:
self.add_zip_feedback(archive, fb, fbs[fb], wk,
l_file_id, l_ret, job_id,
restricted=restricted)
transaction.commit()
else:
result['content'] = r.content
except Exception as e:
# log this
result['content'] = str(e)

return result
61 changes: 45 additions & 16 deletions Products/Reportek/Collection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from Products.Reportek.interfaces import ICollection
from Products.Reportek import DEPLOYMENT_BDR, REPORTEK_DEPLOYMENT
from Products.Reportek.config import permission_manage_properties_collections
from Products.Reportek.catalog import searchResults
from Products.PageTemplates.PageTemplateFile import PageTemplateFile
from OFS.Folder import Folder
from DateTime import DateTime
Expand All @@ -48,9 +49,6 @@
import json
__version__ = '$Revision$'[11:-2]


# product imports

manage_addCollectionForm = PageTemplateFile('zpt/collection/add', globals())


Expand All @@ -73,6 +71,9 @@ def manage_addCollection(self, title, descr, year, endyear, partofyear,
if isinstance(self, Collection):
if allow_referrals != self.are_referrals_allowed():
ob.prop_allowed_referrals = allow_referrals
# If parent collection is restricted, set the child restricted
if self.restricted:
ob.restricted = True

self._setObject(id, ob)
if REQUEST is not None:
Expand Down Expand Up @@ -260,8 +261,9 @@ def get_users_list(self, REQUEST):
'roles': [role]
}
# retrieve the local accounts
folders = catalog(
meta_type=['Report Collection'], path=self.absolute_url(1))
folders = searchResults(catalog,
dict(meta_type=['Report Collection'],
path=self.absolute_url(1)))
for folder in folders:
context = catalog.getobject(folder.data_record_id_)
for member, roles in context.get_local_roles():
Expand Down Expand Up @@ -688,14 +690,16 @@ def aggregated_licences(self, all_years=False):
if not isinstance(data, dict):
res["result"] = "Fail"
res["message"] = "Malformed body"
if all_years:
year = ''
elif 'year' in data:
year = str(data.get('year'))
del data['year']
else:
# Default to the previous year
year = str(DateTime().year() - 1)
year = self.REQUEST.get('year')
if not year:
if all_years:
year = ''
elif 'year' in data:
year = str(data.get('year'))
del data['year']
else:
# Default to the previous year
year = str(DateTime().year() - 1)
response = registry.get_company_licences(
self.company_id,
domain=domain,
Expand All @@ -712,6 +716,10 @@ def aggregated_licences(self, all_years=False):
else:
res["result"] = "Fail"
res["message"] = None
# FLOAT_REPR is deprecated in python 3.6
json.encoder.FLOAT_REPR = (
lambda o: ("%.7f" % o).rstrip("0") if o != int(o) else str(o)
)

return json.dumps(res, indent=4)

Expand Down Expand Up @@ -1048,7 +1056,7 @@ def get_released_envelopes(self):
'sort_order': 'reverse',
'path': path
}
envs = self.Catalog(**query)
envs = searchResults(self.Catalog, query)

return envs

Expand All @@ -1059,7 +1067,7 @@ def get_latest_env_reportingdate(self):
'sort_order': 'reverse',
'path': path
}
envs = self.Catalog(**query)
envs = searchResults(self.Catalog, query)
if envs:
return envs[0].reportingdate

Expand All @@ -1073,7 +1081,7 @@ def is_newest_released(self, env_id):

@property
def Description(self):
if isinstance(self.descr, unicode):
if isinstance(self.descr, unicode): # noqa: F821
return self.descr.encode('utf-8')

return self.descr
Expand All @@ -1084,5 +1092,26 @@ def add_envelope(self, **kwargs):
self.manage_addEnvelope,
**kwargs)

security.declareProtected('View management screens', 'set_restricted')

def set_restricted(self, permission, roles, acquire=0, REQUEST=None):
"""
Restrict access to the named objects.
Figure out what roles exist, but don't give access to
anonymous and authenticated
"""
self.manage_permission(permission_to_manage=permission,
roles=roles, acquire=acquire)
self.restricted = True
self.reindex_object()

@property
def restricted(self):
return getattr(self, '_restricted', False)

@restricted.setter
def restricted(self, value):
self._restricted = bool(value)


Globals.InitializeClass(Collection)
3 changes: 2 additions & 1 deletion Products/Reportek/DataflowMappings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from AccessControl import ClassSecurityInfo
from AccessControl.Permissions import view_management_screens
from constants import DATAFLOW_MAPPINGS, ENGINE_ID
from Products.Reportek.catalog import searchResults
from DataflowMappingsRecord import DataflowMappingsRecord
from Globals import InitializeClass
from OFS.Folder import Folder
Expand Down Expand Up @@ -49,7 +50,7 @@ def get_dataflow_mapping_records(self, dataflow_uris, web_form_only):
else:
query['dataflow_uri'] = [dataflow_uris]

return self.Catalog(**query)
return searchResults(self.Catalog, query)

def getSchemaObjectsForDataflows(self, dataflow_uris, web_form_only):
"""
Expand Down
9 changes: 5 additions & 4 deletions Products/Reportek/DataflowMappingsRecord.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from Products.Five.browser import BrowserView
from Products.PageTemplates.PageTemplateFile import PageTemplateFile
from Products.ZCatalog.CatalogAwareness import CatalogAware
from Products.Reportek.catalog import searchResults
from ZODB.PersistentList import PersistentList
__doc__ = """ Multiple dataflow mappings for a single obligation """
log = logging.getLogger(__name__)
Expand All @@ -33,10 +34,10 @@ def add(self):
self.parent.absolute_url() + '/manage_main')

def get_records_by_dataflow(self, dataflow_uri):
return self.parent.Catalog(
meta_type='Dataflow Mappings Record',
dataflow_uri=dataflow_uri,
path='/DataflowMappings')
return searchResults(self.parent.Catalog,
dict(meta_type='Dataflow Mappings Record',
dataflow_uri=dataflow_uri,
path='/DataflowMappings'))

def __call__(self, *args, **kwargs):
if self.request.method == 'POST':
Expand Down
10 changes: 7 additions & 3 deletions Products/Reportek/Document.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@
from OFS.SimpleItem import SimpleItem
from Products.PageTemplates.PageTemplateFile import PageTemplateFile
from Products.Reportek.RepUtils import DFlowCatalogAware, parse_uri
from Products.Reportek.catalog import searchResults
from Products.Reportek.constants import DEFAULT_CATALOG
from Products.ZCatalog.CatalogAwareness import CatalogAware
from webdav.common import rfc1123_date
from XMLInfoParser import SchemaError, detect_schema
Expand Down Expand Up @@ -459,9 +461,11 @@ def get_size(self):
def getFeedbacksForDocument(self):
""" Returns the Feedback objects associated with this document """
fbs = []
brains = self.Catalog(meta_type='Report Feedback',
document_id=self.id,
path=self.getParentNode().absolute_url(1))
brains = searchResults(
self.getPhysicalRoot().unrestrictedTraverse(DEFAULT_CATALOG),
dict(meta_type='Report Feedback',
document_id=self.id,
path=self.getParentNode().absolute_url(1)))
for brain in brains:
try:
fbs.append(brain.getObject())
Expand Down
Loading

0 comments on commit 0994450

Please sign in to comment.