From 21524a89acc29482a7cd0e31e79f51df03080af1 Mon Sep 17 00:00:00 2001 From: dvasani-CS Date: Thu, 23 Mar 2023 14:55:09 +0530 Subject: [PATCH 1/2] added profiles client --- cortex_common/__init__.py | 15 + cortex_common/constants/__init__.py | 18 + cortex_common/constants/contexts.py | 121 ++++ cortex_common/constants/queries.py | 382 +++++++++++++ cortex_common/constants/types.py | 54 ++ cortex_common/types/__init__.py | 24 + cortex_common/types/api.py | 53 ++ cortex_common/types/attributes.py | 179 ++++++ cortex_common/types/common.py | 131 +++++ cortex_common/types/schemas.py | 262 +++++++++ cortex_common/utils/__init__.py | 62 +++ cortex_common/utils/assertion_utils.py | 83 +++ cortex_common/utils/attr_utils.py | 412 ++++++++++++++ cortex_common/utils/class_utils.py | 59 ++ cortex_common/utils/config_utils.py | 109 ++++ cortex_common/utils/dataframe_utils.py | 357 ++++++++++++ cortex_common/utils/equality_utils.py | 129 +++++ cortex_common/utils/etl_utils.py | 15 + cortex_common/utils/generator_utils.py | 126 +++++ cortex_common/utils/id_utils.py | 30 + cortex_common/utils/object_utils.py | 587 ++++++++++++++++++++ cortex_common/utils/string_utils.py | 74 +++ cortex_common/utils/time_utils.py | 265 +++++++++ cortex_common/utils/type_utils.py | 156 ++++++ cortex_common/validators/__init__.py | 17 + cortex_common/validators/lists.py | 63 +++ cortex_profiles/__init__.py | 18 + cortex_profiles/datamodel/__init__.py | 15 + cortex_profiles/datamodel/constants.py | 88 +++ cortex_profiles/datamodel/dataframes.py | 50 ++ cortex_profiles/ext/__init__.py | 75 +++ cortex_profiles/ext/builders.py | 142 +++++ cortex_profiles/ext/clients.py | 205 +++++++ cortex_profiles/ext/rest.py | 704 ++++++++++++++++++++++++ requirements-dev.txt | 5 +- 35 files changed, 5084 insertions(+), 1 deletion(-) create mode 100644 cortex_common/__init__.py create mode 100644 cortex_common/constants/__init__.py create mode 100644 cortex_common/constants/contexts.py create mode 100644 cortex_common/constants/queries.py create mode 100644 cortex_common/constants/types.py create mode 100644 cortex_common/types/__init__.py create mode 100644 cortex_common/types/api.py create mode 100644 cortex_common/types/attributes.py create mode 100644 cortex_common/types/common.py create mode 100644 cortex_common/types/schemas.py create mode 100644 cortex_common/utils/__init__.py create mode 100644 cortex_common/utils/assertion_utils.py create mode 100644 cortex_common/utils/attr_utils.py create mode 100644 cortex_common/utils/class_utils.py create mode 100644 cortex_common/utils/config_utils.py create mode 100644 cortex_common/utils/dataframe_utils.py create mode 100644 cortex_common/utils/equality_utils.py create mode 100644 cortex_common/utils/etl_utils.py create mode 100644 cortex_common/utils/generator_utils.py create mode 100644 cortex_common/utils/id_utils.py create mode 100644 cortex_common/utils/object_utils.py create mode 100644 cortex_common/utils/string_utils.py create mode 100644 cortex_common/utils/time_utils.py create mode 100644 cortex_common/utils/type_utils.py create mode 100644 cortex_common/validators/__init__.py create mode 100644 cortex_common/validators/lists.py create mode 100644 cortex_profiles/__init__.py create mode 100644 cortex_profiles/datamodel/__init__.py create mode 100644 cortex_profiles/datamodel/constants.py create mode 100644 cortex_profiles/datamodel/dataframes.py create mode 100644 cortex_profiles/ext/__init__.py create mode 100644 cortex_profiles/ext/builders.py create mode 100644 cortex_profiles/ext/clients.py create mode 100644 cortex_profiles/ext/rest.py diff --git a/cortex_common/__init__.py b/cortex_common/__init__.py new file mode 100644 index 0000000..729fa82 --- /dev/null +++ b/cortex_common/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/cortex_common/constants/__init__.py b/cortex_common/constants/__init__.py new file mode 100644 index 0000000..b37d23c --- /dev/null +++ b/cortex_common/constants/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from .contexts import * +from .types import * diff --git a/cortex_common/constants/contexts.py b/cortex_common/constants/contexts.py new file mode 100644 index 0000000..3b4a424 --- /dev/null +++ b/cortex_common/constants/contexts.py @@ -0,0 +1,121 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from cortex_common.utils.config_utils import AttrsAsDict + +# pylint: disable=too-few-public-methods +__all__ = [ + "ProfileAttributeClassifications", + "SchemaContexts", + "ATTRIBUTES", + "AttributeValues", + "CONTEXTS", +] + + +class SchemaContexts(AttrsAsDict): + """ + An "Enum" like class capturing the contexts of classes relevant to profile schemas. + """ + + PROFILE_SCHEMA = "cortex/profile-schema" + PROFILE_ATTRIBUTE_TAG = "cortex/profile-attribute-tag" + PROFILE_ATTRIBUTE_GROUP = "cortex/profile-attribute-group" + PROFILE_ATTRIBUTE_FACET = "cortex/profile-attribute-facet" + PROFILE_ATTRIBUTE_TAXONOMY = "cortex/profile-attribute-taxonomy" + + +class ATTRIBUTES(AttrsAsDict): + """ + An "Enum" like class capturing the contexts of classes relevant to different types of profile attributes. + """ # pylint: disable=line-too-long + + DECLARED_PROFILE_ATTRIBUTE = "cortex/attributes-declared" + OBSERVED_PROFILE_ATTRIBUTE = "cortex/attributes-observed" + INFERRED_PROFILE_ATTRIBUTE = "cortex/attributes-inferred" + ASSIGNED_PROFILE_ATTRIBUTE = "cortex/attributes-assigned" + + +class AttributeValues(AttrsAsDict): + """ + An "Enum" like class capturing the contexts of classes relevant to different types of values that can be captured in + profile attributes. + """ # pylint: disable=line-too-long + + STRING_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-string" + BOOLEAN_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-boolean" + NUMBER_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-number" + WEIGHT_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-weight" + DATETIME_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-datetime" + + TOTAL_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-total" + STATISTICAL_SUMMARY_ATTRIBUTE_VALUE = "cortex/attribute-value-statsummary" + PERCENTILE_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-percentile" + PERCENTAGE_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-percentage" + + DIMENSIONAL_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-dimensional" + LIST_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-list" + + ENTITY_ATTRIBUTE_VALUE = "cortex/attribute-value-entity" + ENTITY_REL_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-entity-rel" + PROFILE_REL_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-profile-rel" + + # Depricated ... + + COUNTER_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-counter" + INTEGER_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-integer" + DECIMAL_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-decimal" + + # CLASSIFICATION_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-classification" + # RELATIONSHIP_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-relationship" + # INSIGHT_ATTRIBUTE_VALUE = "cortex/attribute-value-insight" + # WEIGHTED_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-weighted" + # PRIMITIVE_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-primitive" + # OBJECT_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-object" + # AVERAGE_PROFILE_ATTRIBUTE_VALUE = "cortex/attribute-value-average" + # CONCEPT_ATTRIBUTE_VALUE = "cortex/attribute-value-concept" + + +class CONTEXTS(AttrsAsDict): + """ + An "Enum" like class capturing the contexts of general classes + """ + + PROFILE = "cortex/profile" + PROFILE_LINK = "cortex/profile-link" + PROFILE_ATTRIBUTE_HISTORIC = "cortex/profile-attribute-historic" + LINK = "cortex/link" + + SESSION = "cortex/session" + INSIGHT = "cortex/insight" + CANDIDATE_INSIGHT = "cortex/candidate-insight" + + INSIGHT_CONCEPT_TAG = "cortex/insight-concept-tag" + INSIGHT_TAG_RELATIONSHIP = "cortex/insight-concept-relationship" + INSIGHT_TAG_RELATED_TO_RELATIONSHIP = "cortex/insight-relatedTo-concept" + INTERACTION = "cortex/interaction" + INSIGHT_INTERACTION = "cortex/insight-interaction" + + +class ProfileAttributeClassifications(AttrsAsDict): + """ + An "Enum" like class capturing the different classifications of attributes. + """ + + inferred = "inferred" + declared = "declared" + observed = "observed" + assigned = "assigned" diff --git a/cortex_common/constants/queries.py b/cortex_common/constants/queries.py new file mode 100644 index 0000000..183695a --- /dev/null +++ b/cortex_common/constants/queries.py @@ -0,0 +1,382 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +# pylint: disable=line-too-long +PROFILE_SCHEMA_COMMON = """ + name + title + names { title categories singular plural} + description + primarySource { + attributes + name + profileGroup + profileKey + timestamp { + auto + field + fixed { + format + value + } + format + } + } + joins { + attributes + join { + joinSourceColumn + primarySourceColumn + } + name + profileGroup + timestamp { + auto + field + fixed { + format + value + } + format + } + } + bucketAttributes { + name + profileGroup + source { + name + } + buckets { + filter + name + } + } + customAttributes { + expression + name + profileGroup + source { + name + } + window + } +""" + +PROFILE_SCHEMA_SUMMARY = ( + """query profileSchemaByName($project: String!, $name: String!) { + profileSchemaByName(project: $project, name: $name) {%s}}""" + % PROFILE_SCHEMA_COMMON +) + +FIND_SCHEMAS = ( + """query profileSchemas($project: String!) { + profileSchemas(project: $project) {%s}}""" + % PROFILE_SCHEMA_COMMON +) + +FIND_PROFILES = """query ProfileList( + $attributes: [String!] + $filter: String + $limit: Int + $profileSchema: String! + $project: String! +) { + profiles( + attributes: $attributes + filter: $filter + limit: $limit + profileSchema: $profileSchema + project: $project + ) { + attributes { + group + timestamp + key + source + type + value + } + profileID + profileSchema + } +}""" + +PROFILES_FOR_PLAN = """ +query profilesForPlan( + $project: String! + $simulationId: String! + $profileSchema: String! + $planId: String! + $filter: String + $limit: Int +) { + profilesForPlan( + project: $project + simulationId: $simulationId + profileSchema: $profileSchema + planId: $planId + filter: $filter + limit: $limit + ) { + profileID + profileSchema + attributes { + group + key + source + timestamp + type + value + } + } +} +""" +PROFILE_BY_ID = """query ProfileViewer($project: String!, $schema: String!, $profile: ID!) { + profile: profileById( + project: $project + profileSchema: $schema + id: $profile + ) { + attributes { + group + key + source + timestamp + type + value + } + profileID + profileSchema + } +} +""" + +PROFILE_GROUP_COUNT = """query ProfileGroupCount($project: String!, $schema: String!, $filter: String, +$groupBy: [String!]!, $limit: Int) { + profileGroupCount(project: $project, profileSchema: $schema, filter: $filter, groupBy: $groupBy, limit: $limit) { + key + count + } +} +""" + +PROFILE_COUNT = """query profileCount($project: String!, $schema: String!, $filter: String) { + profileCount(project: $project, profileSchema: $schema, filter: $filter) + } + """ + +PROFILE_FEATURES = """query ProfileFeatures($project: String!, $profileSchema: String!) { + profileFeatures(project: $project, profileSchema: $profileSchema) { + dataType + featureName + featureType + maxValue + meanValue + minValue + pctDom + pctNull + profileGroup + observations + stdDev + uniqueCount + } +} +""" + +PROFILE_HISTORY = """query ProfileHistory($project: String!, $profileSchema: String!, $limit: Int) { + profileHistory(project: $project, profileSchema: $profileSchema, limit: $limit) { + profileSchema + project + commitInfo{ + clusterId + isBlindAppend + isolationLevel + operation + operationMetrics + operationParams + readVersion + timestamp + userId + userMetadata + version + } + } +} +""" + +DELETE_PROFILE_SCHEMA = """ +mutation DeleteProfileSchema($input: DeleteProfileSchemaInput!){ + deleteProfileSchema(input: $input) +} +""" + +DELETE_PROFILES = """ +mutation DeleteProfiles($project: String!, $profileSchema: String!, $filter: String!){ + deleteProfiles(project: $project, profileSchema: $profileSchema, filter: $filter){ + endTime + errorMessage + isActive + isCancelled + isComplete + isError + jobId + jobType + project + resourceName + resourceType + startTime + } +} +""" + +DELETE_ALL_PROFILES = """ +mutation DeleteAllProfiles($project: String!, $profileSchema: String!){ + deleteAllProfiles(project: $project, profileSchema: $profileSchema){ + endTime + errorMessage + isActive + isCancelled + isComplete + isError + jobId + jobType + project + resourceName + resourceType + startTime + } +} +""" + +CREATE_PROFILE_SCHEMA = ( + """ +mutation CreateProfileSchema($input: ProfileSchemaInput!) { + createProfileSchema(input: $input) { + %s + } +} +""" + % PROFILE_SCHEMA_COMMON +) + +UPDATE_PROFILE_SCHEMA = ( + """ +mutation UpdateProfileSchema($input: ProfileSchemaInput!) { + updateProfileSchema(input: $input) { + %s + } +} +""" + % PROFILE_SCHEMA_COMMON +) + +CREATE_BUCKET_ATTRIBUTE = ( + """ +mutation CreateBucketAttribute($input: CreateBucketAttributeInput!) { + createBucketAttribute(input: $input){%s} +} +""" + % PROFILE_SCHEMA_COMMON +) + +CREATE_CUSTOM_ATTRIBUTE = ( + """ +mutation CreateCustomAttribute($input: CreateCustomAttributeInput!) { + createCustomAttribute(input: $input) { + %s + } +} +""" + % PROFILE_SCHEMA_COMMON +) + +BUILD_PROFILES_FROM_SCHEMA = """ +mutation BuildProfiles($project: String!, $profileSchema: String!){ + buildProfile(project: $project, profileSchema: $profileSchema){ + endTime + errorMessage + isActive + isCancelled + isComplete + isError + jobId + jobType + project + resourceName + resourceType + startTime + } +} +""" +UPDATE_BUCKET_ATTRIBUTE = ( + """ +mutation UpdateBucketAttribute($input: UpdateBucketAttributeInput!) { + updateBucketAttribute(input: $input){%s} +} +""" + % PROFILE_SCHEMA_COMMON +) + +UPDATE_CUSTOM_ATTRIBUTE = ( + """ +mutation UpdateCustomAttribute($input: UpdateCustomAttributeInput!) { + updateCustomAttribute(input: $input) { + %s + } +} +""" + % PROFILE_SCHEMA_COMMON +) + +UPDATE_PROFILES = """ +mutation UpdateProfiles($project: String!, $profileSchema: String!, $profiles:[String!]!){ + updateProfiles(project: $project, profileSchema: $profileSchema, profiles: $profiles){ + endTime + errorMessage + isActive + isCancelled + isComplete + isError + jobId + jobType + project + resourceName + resourceType + startTime + } +} +""" + +DELETE_BUCKET_ATTRIBUTE = ( + """ +mutation DeleteBucketAttribute($input: DeleteBucketAttributeInput!) { + deleteBucketAttribute(input: $input){%s} +} +""" + % PROFILE_SCHEMA_COMMON +) + +DELETE_CUSTOM_ATTRIBUTE = ( + """ +mutation DeleteCustomAttribute($input: DeleteCustomAttributeInput!) { + deleteCustomAttribute(input: $input) { + %s + } +} +""" + % PROFILE_SCHEMA_COMMON +) + +# pylint: enable=line-too-long diff --git a/cortex_common/constants/types.py b/cortex_common/constants/types.py new file mode 100644 index 0000000..2341d8b --- /dev/null +++ b/cortex_common/constants/types.py @@ -0,0 +1,54 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from cortex_common.utils.config_utils import AttrsAsDict + + +__all__ = [ + "VERSION", + "CAMEL", + "DESCRIPTIONS", +] + + +VERSION = "0.0.1" +CAMEL = "2.0.0" + + +# pylint: disable=too-few-public-methods + +class DESCRIPTIONS(AttrsAsDict): + """ + An "Enum" like class capturing the descriptions of common fields used in classes. + """ + + ID = "How can this piece of data be uniquely identified?" + CONTEXT = "What is the type of this piece of data?" + VERSION = "What version of the CAMEL spec is this piece of data based on?" + ATTRIBUTE_SUMMARY = "How can the value of this attribute be concisely expressed?" + ATTRIBUTE_KEY = "What is the unqiue key for the attribute that distinguishes it from the rest of the attributes captured w.r.t the profile?" # pylint: disable=line-too-long + CREATED_AT = "When was this piece of data created?" + UPDATED_AT = "When was this piece of data created?" + CAMEL = "What CAMEL spec does this piece of data adhere to?" + NAME = "What is a unqiue name for this piece of data?" + LABEL = "What is a UI friendly short name for this piece of data?" + TITLE = "What is a UI friendly short name for this piece of data?" + DESCRIPTION = ( + "What is the detailed explanation of the purpose of this piece of data?" + ) + TAGS = "What tags are applicable to this CAMEL resource?" + WEIGHT = "How relevant is this piece of information?" + PROJECT = "what is the project name?" diff --git a/cortex_common/types/__init__.py b/cortex_common/types/__init__.py new file mode 100644 index 0000000..8857c4f --- /dev/null +++ b/cortex_common/types/__init__.py @@ -0,0 +1,24 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# ----- Order Here May Be Important! ----- +from .api import * +from .schemas import * + +from .attributes import * +from .common import * + +# ----- --------------------------- ------ diff --git a/cortex_common/types/api.py b/cortex_common/types/api.py new file mode 100644 index 0000000..9fa48f3 --- /dev/null +++ b/cortex_common/types/api.py @@ -0,0 +1,53 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import Optional + +from attr import attrs + +from cortex_common.utils.attr_utils import describableAttrib + +__all__ = [ + "MessageResponse", + "ErrorResponse", +] + + +@attrs(frozen=True) +class MessageResponse: + """ + General Success Message Returned from the Cortex APIs + """ + + message = describableAttrib( + type=str, description="What is the status of the version increment request?" + ) + version = describableAttrib( + type=Optional[int], + default=None, + description="What is the current version of the resource?", + ) + + +@attrs(frozen=True) +class ErrorResponse: + """ + General Error Message Returned from the Cortex APIs + """ + + error = describableAttrib( + type=str, description="What is the error message associated with the request?" + ) diff --git a/cortex_common/types/attributes.py b/cortex_common/types/attributes.py new file mode 100644 index 0000000..46d81ce --- /dev/null +++ b/cortex_common/types/attributes.py @@ -0,0 +1,179 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +# pylint: disable=cyclic-import +from typing import Optional, List + +from attr import attrs + +from cortex_common.constants.types import DESCRIPTIONS +from cortex_common.utils.attr_utils import describableAttrib, attr_class_to_dict, dicts_to_classes +from cortex_common.types.common import ResourceRef + +__all__ = [ + "ProfileFeature", + "ProfileAttribute", + "BucketAttributeSpec", + "CustomAttributeSpec", + "BucketSpec", +] + + +@attrs(frozen=True) +class ProfileAttribute: + """ + Represents the profile attribute ... + """ + + group = describableAttrib( + type=str, description="What is the group of profile attribute?" + ) + key = describableAttrib(type=str, description=DESCRIPTIONS.ATTRIBUTE_KEY) + source = describableAttrib( + type=str, description="What is the source of this attribute?" + ) + timestamp = describableAttrib( + type=str, description="When is this Attribute created?" + ) + type = describableAttrib( + type=str, description="What is the type of this attribute?" + ) + value = describableAttrib( + type=str, description="What is the value of this attribute?" + ) + + +@attrs(frozen=True) +class ProfileFeature: + """ + Represents the Profile Feature Summary + """ + + dataType = describableAttrib(type=str, default=None) + description = describableAttrib( + type=str, default=None, description=DESCRIPTIONS.DESCRIPTION + ) + featureName = describableAttrib(type=str, default=None) + featureType = describableAttrib(type=str, default=None) + maxValue = describableAttrib( + type=float, default=None, description="Max Value of the Attribute" + ) + meanValue = describableAttrib( + type=float, default=None, description="Mean Value of the Attribute" + ) + minValue = describableAttrib( + type=float, default=None, description="Min Value of the Attribute" + ) + notes = describableAttrib(type=str, default=None) + observations = describableAttrib(type=str, default=None) + pctDom = describableAttrib(type=float, default=None) + pctNull = describableAttrib(type=float, default=None) + profileGroup = describableAttrib( + type=str, default=None, description="Profile Group of the Attribute" + ) + projectName = describableAttrib( + type=str, default=None, description=DESCRIPTIONS.PROJECT + ) + sourceName = describableAttrib(type=str, default=None) + stdDev = describableAttrib(type=float, default=None) + tableName = describableAttrib(type=str, default=None) + timestamp = describableAttrib(type=str, default=None) + uniqueCount = describableAttrib(type=str, default=None) + + def __iter__(self): + return iter(attr_class_to_dict(self, skip_nulls=True).items()) + + def to_dict(self): + """ + to_dict + :return: + :rtype: + """ + return attr_class_to_dict(self, skip_when_serializing=False, skip_nulls=True) + + +@attrs(frozen=True) +class AttributeSpec: + """ + AttributeSpec class + """ + name = describableAttrib(type=str, description="Attribute Name") + profileGroup = describableAttrib( + type=Optional[str], description="Profile Group of the Calculated Attribute" + ) + source = describableAttrib( + type=ResourceRef, description="Source of the Calculated Attribute" + ) + + +@attrs(frozen=True) +class BucketSpec: + """ + BucketSpec class + """ + name = describableAttrib(type=str, description="Bucket Name") + filter = describableAttrib(type=str, description="Bucket Filter") + + +@attrs(frozen=True) +class BucketAttributeSpec(AttributeSpec): + """ + BucketAttributeSpec class + """ + buckets = describableAttrib( + type=List[BucketSpec], + converter=lambda l: dicts_to_classes(l, BucketSpec), + description="Buckets for a Bucketed Attribute", + ) + + def __iter__(self): + """ + __iter__ + :return: + :rtype: + """ + return iter(attr_class_to_dict(self, skip_nulls=True).items()) + + def to_dict(self): + """ + to_dict + :return: + :rtype: + """ + return attr_class_to_dict(self, skip_when_serializing=False, skip_nulls=True) + + +@attrs(frozen=True) +class CustomAttributeSpec(AttributeSpec): + """ + CustomAttributeSpec class + """ + expression = describableAttrib( + type=str, description="Expression of the Custom Attribute" + ) + window = describableAttrib( + type=Optional[str], description="Window of the Custom Attribute" + ) + + def __iter__(self): + return iter(attr_class_to_dict(self, skip_nulls=True).items()) + + def to_dict(self): + """ + to_dict + :return: + :rtype: + """ + return attr_class_to_dict(self, skip_when_serializing=False, skip_nulls=True) diff --git a/cortex_common/types/common.py b/cortex_common/types/common.py new file mode 100644 index 0000000..1cc2aad --- /dev/null +++ b/cortex_common/types/common.py @@ -0,0 +1,131 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +# pylint: disable=cyclic-import +from typing import Optional + +from attr import attrs + +from cortex_common.utils.attr_utils import describableAttrib, dict_to_attr_class + +__all__ = [ + "JobInfo", + "TimestampSpec", + "FixedTimestampSpec", + "CommitInfo", + "ResourceRef", +] + + +@attrs(frozen=True) +class JobInfo: + """ + JobInfo class + """ + endTime = describableAttrib( + type=str, default=None, description="End Time of the Job" + ) + errorMessage = describableAttrib( + type=str, default=None, description="Job Error Message" + ) + isActive = describableAttrib( + type=bool, default=False, description="Is the Job still running?" + ) + isCancelled = describableAttrib( + type=bool, default=False, description="Is the Job Cancelled?" + ) + isComplete = describableAttrib( + type=bool, default=False, description="Does the Job Complete?" + ) + isError = describableAttrib( + type=bool, default=False, description="Does the Job fail?" + ) + jobId = describableAttrib(type=str, default=None, description="Job Identifier") + jobType = describableAttrib(type=str, default=None, description="Type of Job") + project = describableAttrib(type=str, default=None, description="Project Name") + resourceName = describableAttrib( + type=str, default=None, description="Resource Name" + ) + resourceType = describableAttrib( + type=str, default=None, description="Resource Type" + ) + startTime = describableAttrib( + type=str, default=None, description="Start Time of the Job" + ) + + +@attrs(frozen=True) +class FixedTimestampSpec: + """ + Represents the type of a value an attribute can hold + """ + + value = describableAttrib(type=Optional[str], description="Value of the Timestamp") + format = describableAttrib( + type=Optional[str], description="Format of the Timestamp" + ) + + +@attrs(frozen=True) +class TimestampSpec: + """ + Represents the type of a value an attribute can hold + """ + + auto = describableAttrib( + type=Optional[bool], + default=None, + description="Is the Timestamp Auto Generated?", + ) + fixed = describableAttrib( + type=Optional[FixedTimestampSpec], + default=None, + converter=lambda l: dict_to_attr_class(l, FixedTimestampSpec), + description="Fixed Timestamp Value and Format", + ) + field = describableAttrib( + type=Optional[str], default=None, description="Field of the timestamp" + ) + format = describableAttrib( + type=Optional[str], default=None, description="Format of the timestamp" + ) + + +@attrs(frozen=True) +class CommitInfo: + """ + Represents the type of a value an attribute can hold + """ + + clusterId = describableAttrib(type=str, default=None) + isBlindAppend = describableAttrib(type=bool, default=None) + isolationLevel = describableAttrib(type=str, default=None) + operation = describableAttrib(type=str, default=None) + operationMetrics = describableAttrib(type=dict, default=None) + operationParams = describableAttrib(type=dict, default=None) + readVersion = describableAttrib(type=int, default=None) + timestamp = describableAttrib(type=str, default=None) + userId = describableAttrib(type=str, default=None) + userMetadata = describableAttrib(type=str, default=None) + version = describableAttrib(type=int, default=None) + + +@attrs(frozen=True) +class ResourceRef: + """ + Represents the type of a value an attribute can hold + """ + + name = describableAttrib(type=str, description="Resource Name") diff --git a/cortex_common/types/schemas.py b/cortex_common/types/schemas.py new file mode 100644 index 0000000..f490388 --- /dev/null +++ b/cortex_common/types/schemas.py @@ -0,0 +1,262 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +# pylint: disable=cyclic-import +from typing import List, Optional + +from attr import attrs + +from cortex_common.constants.types import DESCRIPTIONS +from cortex_common.types.attributes import ( + CustomAttributeSpec, + BucketAttributeSpec, + ProfileAttribute, +) +from cortex_common.types.common import TimestampSpec, CommitInfo +from cortex_common.utils.attr_utils import ( + describableAttrib, + dicts_to_classes, + attr_class_to_dict, + dict_to_attr_class, +) + +__all__ = [ + "ProfileSchema", + "ProfileNames", + "DataSourceSelection", + "GroupCount", + "ProfileCommit", + "JoinSourceSelection", + "Profile", + "JoinSpec", +] + + +@attrs(frozen=True) +class JoinSpec: + """ + JoinSpec class + """ + joinSourceColumn = describableAttrib( + type=str, description="Secondary source key to Join the Columns" + ) + primarySourceColumn = describableAttrib( + type=str, description="Primary source key to Join the Columns" + ) + joinType = describableAttrib(type=str, default=None, description="Type of Join") + + +@attrs(frozen=True) +class JoinSourceSelection: + """ + JoinSourceSelection class + """ + attributes = describableAttrib( + type=Optional[List[str]], description="Attributes selected from this source" + ) + join = describableAttrib( + type=JoinSpec, + converter=lambda l: dict_to_attr_class(l, JoinSpec), + description="Join Specification", + ) + name = describableAttrib(type=str, description="Name of the Joined Data Source") + profileGroup = describableAttrib( + type=Optional[str], description="Group of this source" + ) + timestamp = describableAttrib( + type=Optional[TimestampSpec], + converter=lambda l: dict_to_attr_class(l, TimestampSpec), + description="Timestamp", + ) + + +@attrs(frozen=True) +class DataSourceSelection: + """ + Represents the type of a value an attribute can hold + """ + + name = describableAttrib( + type=str, description="Name of the Primary Source of Profile Schema" + ) + attributes = describableAttrib( + type=Optional[List[str]], + description="Attributes selected from the Primary source", + ) + profileKey = describableAttrib( + type=str, description="profile key of the Primary Source(ProfileID)" + ) + timestamp = describableAttrib( + type=Optional[TimestampSpec], + converter=lambda l: dict_to_attr_class(l, TimestampSpec), + description="Timestamp", + ) + profileGroup = describableAttrib( + type=Optional[str], default=None, description="Group of this primary source" + ) + + +@attrs(frozen=True) +class ProfileNames: + """ + Represents the type of a value an attribute can hold + """ + + categories = describableAttrib( + type=Optional[List[str]], description="Categories of the Profile Schema" + ) + singular = describableAttrib( + type=str, description="Singular Name of the Profile Schema" + ) + plural = describableAttrib( + type=str, description="Plural Name of the Profile Schema" + ) + title = describableAttrib(type=str, description="Title of the Profile Schema") + + +@attrs(frozen=True) +class GroupCount: + """ + Represents the type of a value an attribute can hold + """ + + count = describableAttrib( + type=int, + default=None, + description="Count of Profiles based on Specified Grouping", + ) + key = describableAttrib( + type=str, default=None, description="Key in a grouped Attribute" + ) + + +@attrs(frozen=True) +class ProfileCommit: + """ + Represents the type of a value an attribute can hold + """ + + profileSchema = describableAttrib( + type=str, default=None, description="Name of the Profile Schema" + ) + project = describableAttrib( + type=str, default=None, description="Project of the Profile Schema" + ) + commitInfo = describableAttrib( + type=CommitInfo, + default=None, + converter=lambda l: dict_to_attr_class(l, CommitInfo), + description="Commit Info of the Profile Schema", + ) + + +@attrs(frozen=True) +class ProfileSchema: + """ + Represents a group of attributes shared by a class of entities. + """ + + # ---- + name = describableAttrib(type=str, default=None, description=DESCRIPTIONS.NAME) + title = describableAttrib(type=str, default=None, description=DESCRIPTIONS.TITLE) + description = describableAttrib( + type=str, default=None, description=DESCRIPTIONS.DESCRIPTION + ) + names = describableAttrib( + type=ProfileNames, + default=None, + converter=lambda l: dict_to_attr_class(l, ProfileNames), + description="Names of the profile schema", + ) + primarySource = describableAttrib( + type=DataSourceSelection, + default=None, + converter=lambda l: dict_to_attr_class(l, DataSourceSelection), + description="Primary Source the profile schema", + ) + project = describableAttrib( + type=str, default=None, description=DESCRIPTIONS.PROJECT + ) + + joins = describableAttrib( + type=List[JoinSourceSelection], + converter=lambda l: dicts_to_classes(l, JoinSourceSelection), + description="How Schemas are Joined together?", + default=None, + ) + + customAttributes = describableAttrib( + type=List[CustomAttributeSpec], + converter=lambda l: dicts_to_classes(l, CustomAttributeSpec), + description="Custom Attributes in a profile schema", + default=None, + ) + bucketAttributes = describableAttrib( + type=List[BucketAttributeSpec], + converter=lambda l: dicts_to_classes(l, BucketAttributeSpec), + description="Bucket Attributes in a profile schema", + default=None, + ) + + def __iter__(self): + """ + __iter__ + :return: + :rtype: + """ + return iter(attr_class_to_dict(self, skip_nulls=True).items()) + + def to_dict(self): + """ + to_dict + :return: + :rtype: + """ + return attr_class_to_dict(self, skip_when_serializing=False, skip_nulls=True) + + +@attrs(frozen=True) +class Profile: + """ + Profile Representation... + """ + + profileID = describableAttrib( + type=str, description="What is the id for this profile?" + ) + profileSchema = describableAttrib( + type=str, description="What is the id of the schema applied to this profile?" + ) + attributes = describableAttrib( + type=List[ProfileAttribute], + converter=lambda l: dicts_to_classes(l, ProfileAttribute), + description="What are the attributes of this profile?", + ) + + def __iter__(self): + """ + __iter__ + :return: + :rtype: + """ + return iter(attr_class_to_dict(self, skip_nulls=True).items()) + + def to_dict(self): + """ + to_dict + :return: + :rtype: + """ + return attr_class_to_dict(self, skip_when_serializing=False, skip_nulls=True) diff --git a/cortex_common/utils/__init__.py b/cortex_common/utils/__init__.py new file mode 100644 index 0000000..78ef3c8 --- /dev/null +++ b/cortex_common/utils/__init__.py @@ -0,0 +1,62 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import logging +import os + + +def get_logger(name, log_level=None): + """ + Gets a logger with the given name. + """ + logger = logging.getLogger(name) + if not logger.hasHandlers(): + log_level = ( + os.environ.get("LOG_LEVEL", logging.getLogger().level) + if log_level is None + else log_level + ) + formatter = logging.Formatter( + "%(asctime)s - %(levelname)s - %(name)s/%(module)s: %(message)s" + ) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(log_level) + return logger + + +# The reason these packages were named all with a suffix of _util was so they don't get confused +# with any builtins ... # pylint: disable=line-too-long +# pylint: disable=wrong-import-position +from .config_utils import * + +from .assertion_utils import * +from .id_utils import * + +from .type_utils import * +from .attr_utils import * +from .class_utils import * + +from .object_utils import * +from .time_utils import * +from .equality_utils import * + +from .string_utils import * +from .generator_utils import * + +# These need to come after the rest ... +from .etl_utils import * +from .dataframe_utils import * diff --git a/cortex_common/utils/assertion_utils.py b/cortex_common/utils/assertion_utils.py new file mode 100644 index 0000000..9e0be8c --- /dev/null +++ b/cortex_common/utils/assertion_utils.py @@ -0,0 +1,83 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import List, Callable, Any + +import pandas as pd + +__all__ = [ + "is_not_none_or_nan", + "all_values_in_list_are_not_nones_or_nans", + "all_values_in_list_pass", + "first_arg_is_type_wrapper", + "key_has_value_in_dict", +] + + +# pylint: disable=invalid-name +# pylint: disable=simplifiable-if-expression +def is_not_none_or_nan(v: object) -> bool: + """ + Asserting an object (usually an element in a df) is not NaN or None + :param v: + :return: + """ + return ( + (True if v else False) + if not isinstance(v, float) + else (not pd.isna(v) if v else False) + ) + + +def all_values_in_list_are_not_nones_or_nans(l: List) -> bool: + """ + Asserts that all vals in a list are set + :param l: + :return: + """ + return all_values_in_list_pass(l, is_not_none_or_nan) + + +def all_values_in_list_pass(l: List, validity_filter: Callable) -> bool: + """ + Assert all times in list pass the validity check function. + :param l: + :param validity_filter: + :return: + """ + return all(map(validity_filter, l)) + + +def first_arg_is_type_wrapper(_callable, tuple_of_types) -> Callable[[Any], bool]: + """ + ??? + :param _callable: + :param tuple_of_types: + :return: + """ + return lambda x: x if not isinstance(x, tuple_of_types) else _callable(x) + + +def key_has_value_in_dict(d: dict, key: str, value: object): + """ + Check if a dictionary has a specific key with a specific value within it. + + :param d: + :param key: + :param value: + :return: + """ + return isinstance(d, dict) and d.get(key) == value diff --git a/cortex_common/utils/attr_utils.py b/cortex_common/utils/attr_utils.py new file mode 100644 index 0000000..d3844cd --- /dev/null +++ b/cortex_common/utils/attr_utils.py @@ -0,0 +1,412 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import copy +import datetime +import traceback +from typing import Union, Optional, List, TypeVar, Type, Any, Callable, Mapping, cast + +import arrow +import attr +import pandas as pd +import pydash + +from cortex.utils import get_logger +from cortex_common.utils.type_utils import is_union_type, get_types_of_union + +log = get_logger(__name__) + +# pylint: disable=line-too-long +# pylint: disable=logging-fstring-interpolation +# pylint: disable=broad-exception-raised +# pylint: disable=broad-exception-caught +# pylint: disable=invalid-name +# pylint: disable=unnecessary-lambda +# pylint: disable=no-else-return +# pylint: disable=too-many-return-statements +# pylint: disable=unnecessary-lambda-assignment +# pylint: disable=inconsistent-return-statements +# pylint: disable=useless-return + + +__all__ = [ + "construct_attr_class_from_dict", + "dict_to_attr_class", + "dicts_to_classes", + "attr_class_to_dict", + "describableAttrib", + "describableStrAttrib", + "str_or_context", + "union_type_validator", + "converter_for_union_type", + "time_converter", + "attr_fields_except", + "keep_fields_for_attr_value", + "modify_attr_class", + "field_names_of_attr_class", + "not_none_validator", + "datetime_to_iso_converter", +] + +NoneType = type(None) +T = TypeVar("T") + + +def construct_attr_class_from_dict(cls: Type[T], data: dict) -> Optional[T]: + """ + Deals with internal variable renaming ... + :param cls: + :param data: + :return: + """ + try: + # Map attributes with underscores ... + # only do this if things are not a union type ... the dict_constructor is required if + # union ... + attr_fields = list(attr.fields_dict(cls).keys()) + private_var_mappings = { + attr_field: attr_field.replace("_", "", 1) + for attr_field in attr_fields + if attr_field.startswith("_") + } + data_fields = pydash.rename_keys( + { + k: v for k, v in data.items() if k in attr_fields + }, # Ignore fields not in schema ... + private_var_mappings, + ) + return_val = cls(**data_fields) # type: ignore # + except Exception as ex: + log.error(traceback.format_exc()) + log.error(f"Could not construct an instance of {cls} from {data}") + raise ex + return return_val + + +def dict_to_attr_class( + data: Union[dict, T], + desired_attr_type: Type[T], + dict_constructor: Optional[Callable] = None, +) -> Optional[T]: + """ + Convert a attribute into an attr class ... + Previously known as `converter_for_classes` + :param data: + :param desired_attr_type: + :param dict_constructor: + :return: + """ + + if is_union_type(desired_attr_type) and dict_constructor is None: + raise Exception( + "Cannot construct union type {}, dict_constructor required.".format( + desired_attr_type + ) + ) + + valid_types = ( + tuple([desired_attr_type]) + if not is_union_type(desired_attr_type) + else get_types_of_union(desired_attr_type) + ) + + if data is None: + return None + + if not isinstance(data, valid_types + tuple([dict])): + raise Exception("Invalid type {} of data.".format(type(data))) + + # Don't construct already valid types ... and consider union types ... + # What happens if one of the union types is a dict??? + if isinstance(data, valid_types) and not isinstance(data, dict): + return data + + # Do we allow data to be "dict" like items? + return ( + construct_attr_class_from_dict(desired_attr_type, cast(dict, data)) + if dict_constructor is None + else dict_constructor(data) + ) + + +def dicts_to_classes( + l: List[Union[dict, T]], cls: Type[T], dict_constructor: Optional[Callable] = None +) -> Optional[List[Optional[T]]]: + """ + Turns a List of Dicts into a List of Class Instances ... + In the case that there are invalid items in the input list, None is returned .. + Previously known as: converter_for_list_of_classes, list_converter + + + :param l: + :param cls: + :return: + """ + + if l is None: + return None + + if not l: + return cast(List[Optional[T]], l) + + # Check to see if there invalid types to be converted in the list ... + valid_types = tuple([cls]) if not is_union_type(cls) else get_types_of_union(cls) + invalid_types_in_list = list( + map( + lambda x: type(x), + filter(lambda x: not isinstance(x, valid_types + tuple([dict])), l), + ) + ) + if invalid_types_in_list: + e = Exception("Invalid type(s) {} in list".format(invalid_types_in_list)) + log.error(e) + return None + + return [ + dict_to_attr_class(elem, cls, dict_constructor=dict_constructor) for elem in l + ] + + +def attr_class_to_dict( + attr_class: Any, + skip_when_serializing: bool = True, + skip_nulls: bool = False, + hide_internal_attributes: bool = False, +) -> dict: + """ + Turns an attr oriented class into a dict, recursively ignoring any fields that have been marked as `skip_when_serializing` + Previously known as attr_instance_to_dict + + :param attr_class: + :return: + """ + # If filter evaluates to true for an attribute ... its kept ... + return attr.asdict( + attr_class, + filter=lambda a, v: ( + ( + not a.metadata.get("skip_when_serializing", False) + if skip_when_serializing + else True + ) + and ( + not a.metadata.get("internal", False) + if hide_internal_attributes + else True + ) + and (v is not None if skip_nulls else True) + ), + ) + + +def describableAttrib( + description: str = None, + skip_when_serializing: Optional[bool] = None, + internal: Optional[bool] = None, + **kwargs, +) -> dict: + """ + An attr.ib helper to create fields on attr classes with more structured metadata. + :param description: + :param skip_when_serializing: + :param internal: + :param kwargs: + :return: + """ + attrib_args = copy.deepcopy(kwargs) + if description: + attrib_args["metadata"] = pydash.merge( + attrib_args.get("metadata", {}), + {"description": description}, + {"internal": internal} if internal is not None else {}, + {"skip_when_serializing": skip_when_serializing} + if skip_when_serializing is not None + else {}, + ) + if internal: + attrib_args["repr"] = False + return attr.attrib(**attrib_args) + + +def describableStrAttrib(*args, **kwargs): + """ + Helper to make an string based attr attribute. + :param args: + :param kwargs: + :return: + """ + return describableAttrib( + *args, type=str, validator=attr.validators.instance_of(str), **kwargs + ) + + +def str_or_context(ip: Union[str, type]) -> Optional[str]: + """ + Given a string ... it keeps it as is ... + Given a type ... it assumes the type is an attr class and attempts to get the default value of the context for the + attr class + :param ip: + :return: + """ + if isinstance(ip, str): + return ip + try: + context = attr.fields(ip).context.default + return context if isinstance(context, str) else None + except Exception as e: + e = TypeError(f"Could not find context for input: {ip}") + log.error(e) + return None + + +def union_type_validator(union_type: type) -> Callable[[Any, Any, Any], bool]: + """ + Returns an attr validator that ensures that the given value is one of the union types + :param union_type: + :return: + """ + + def validator(value) -> bool: + return type(value) in get_types_of_union(union_type) + + return validator + + +def converter_for_union_type( + handlers: Mapping[type, Union[Type[Any], Callable[[Any], Any]]], +) -> Callable[[Any], Optional[T]]: + """ + Returns an attr oriented converter that is capable of converting a value into the appropriate type for a union type + :param handlers: + :return: + """ + + def invalid_input_handler(data: Any) -> NoneType: + error_message = "Can not convert data of type {} into Union[{}], no valid handlers found.".format( + type(data), ",".join(map(str, handlers.keys())) + ) + log.error(error_message) + return None + + def converter(data: Any): + # Previous Bug ... don't assert that the type of the input is in union.__args__ + # ... since union.__args__ types eats up any types that are subclasses + # of each other ... such as int and bool. + assert ( + type(data) in handlers.keys() + ), "Value of unexpected type ({}) encountered. Expecting: {}".format( + type(data), list(handlers.keys()) + ) + return handlers.get(type(data), invalid_input_handler)(data) + + return converter + + +def time_converter(time: Union[str, arrow.arrow.Arrow]) -> Optional[int]: + """ + Converts a time into an epoch with millisecond resolution. + :param time: + :return: + """ + # Assumption that its a utc timestamp ... + if isinstance(time, (str)): + return arrow.get(time).timestamp * 1000 + elif isinstance(time, (arrow.arrow.Arrow)): + return time.timestamp * 1000 + elif isinstance(time, pd.Timestamp): + return time.timestamp() * 1000 + elif isinstance(time, int): + if len(str(time)) == 13: + return time + elif len(str(time)) == 9: + return time * 1000 + else: + log.error(f"Could not convert int time provided {time}") + return None + else: + log.error(f"Type not yet supported for time_converter: {type(time)}") + return None + return None + + +def attr_fields_except( + cls: type, fields_to_ignore: List[attr.Attribute] +) -> List[attr.Attribute]: + """ + List fields of an attr class except for ... + :param cls: + :param fields_to_ignore: + :return: + """ + return [ + attribute for attribute in attr.fields(cls) if attribute not in fields_to_ignore + ] + + +def keep_fields_for_attr_value( + value: type, fields_to_keep: List[attr.Attribute] +) -> Mapping[str, Any]: + """ + Serialize an attr intance into a dict, keeping only specific fields + :param value: + :param fields_to_keep: + :return: + """ + return attr.asdict(value, recurse=False, filter=lambda a, v: a in fields_to_keep) + + +def modify_attr_class(attrClass: type, modifications: dict): + """ + Modify an attr instance, functionally ... + :param attrClass: + :param modifications: + :return: + """ + return attr.evolve(attrClass, **modifications) + + +def field_names_of_attr_class(attr_class: type) -> List[str]: + """ + List all the fields of an attr class + :param attr_class: + :return: + """ + return list(map(lambda x: x.name, attr.fields(attr_class))) + + +not_none_validator = lambda self, a, v: v is not None + + +def datetime_to_iso_converter(d: Union[str, datetime.datetime]) -> str: + """ + Convert a datetime into an iso timestamp ... + :param d: + :return: + """ + if isinstance(d, str): + # Doing this to turn strings like "2019/01/01 into an iso datetime ..." + return str(arrow.get(d)) + if isinstance(d, arrow.arrow.Arrow): + return str(d) + if isinstance(d, datetime.datetime): + return str(arrow.get(d)) + + +if __name__ == "__main__": + # Testing Profile Schema _version conversion ... + from cortex_common.types.schemas import ProfileSchema + + attr.asdict(ProfileSchema(name="1", title="1", description="1", version=1)) diff --git a/cortex_common/utils/class_utils.py b/cortex_common/utils/class_utils.py new file mode 100644 index 0000000..5425c40 --- /dev/null +++ b/cortex_common/utils/class_utils.py @@ -0,0 +1,59 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import Callable, Any +from functools import wraps +from cortex_common.utils.attr_utils import attr_class_to_dict + +# pylint: disable=too-few-public-methods +__all__ = [ + "state_modifier", + "BaseAttrClass", +] + + +def state_modifier(result_factory: Callable, state_updater: Callable[[Any, Any], Any]): + """ + Decorator on class methods that modify the state of the class (self) with the output of the class method + based on the provided state_updater function + :param result_factory: + :param state_updater: + :return: + """ # pylint: disable=line-too-long + + def inner_decorator(f_to_wrap: Callable): + @wraps(result_factory) + def f_that_gets_called(*args, **kwargs): + state_updater(args[0], result_factory(*args[1:], **kwargs)) + return f_to_wrap(args[0]) + + return f_that_gets_called + + return inner_decorator + + +class BaseAttrClass: + """ + Base class for attr oriented models. + """ + + def __iter__(self): + # Skipping nulls ... so that the JS defaults kick into place ... + return iter( + attr_class_to_dict( + self, hide_internal_attributes=True, skip_nulls=True + ).items() + ) diff --git a/cortex_common/utils/config_utils.py b/cortex_common/utils/config_utils.py new file mode 100644 index 0000000..67a6481 --- /dev/null +++ b/cortex_common/utils/config_utils.py @@ -0,0 +1,109 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import base64 + +# pylint: disable=line-too-long +# pylint: disable=no-value-for-parameter +# pylint: disable=bad-mcs-method-argument +# pylint: disable=unnecessary-pass +# pylint: disable=too-few-public-methods + +__all__ = [ + "AttrsAsDict", + "base64encode_string", + "base64decode_string", +] + + +class _AttrsAsDictMeta(type): + """ + Meta class to help transform any python class that extends this type into a dict where the keys are the attributes + of the class and the values are the respected attribute values. + + This class is useful in place of enums, where we want an IDE to auto fill in attributes, but we want to treat the + class like a dict as well. + """ + + def __iter__(self): + """ + __iter__ + :return: + :rtype: + """ + return zip(self.keys(), self.values()) + + def __getitem__(self, arg): + """ + __getitem__ + :param arg: + :type arg: + :return: + :rtype: + """ + return dict(list(self)).get(arg) + + def keys(cls): + """ + keys + :return: + :rtype: + """ + return list(filter(lambda x: x[0] != "_", cls.__dict__.keys())) + + def values(cls): + """ + values + :return: + :rtype: + """ + return [getattr(cls, k) for k in cls.keys()] + + def items(cls): + """ + items + :return: + :rtype: + """ + return dict(list(cls)).items() + + +class AttrsAsDict(metaclass=_AttrsAsDictMeta): + """ + Any class that extends this will have its attributes be transformable into a dict. + """ + + pass + + +def base64encode_string(string_to_base64encode: str, encoding: str = "utf-8") -> str: + """ + Encodes a string into a base64 encoded string + :param base64encoded_jsonstring: + :return: + """ + return base64.urlsafe_b64encode(string_to_base64encode.encode(encoding)).decode( + encoding + ) + + +def base64decode_string(base64encoded_string: str, encoding="utf-8") -> str: + """ + Decodes a base64 encoded string back into the original string. + :param base64encoded_jsonstring: + :return: + """ + return base64.urlsafe_b64decode(base64encoded_string).decode(encoding) diff --git a/cortex_common/utils/dataframe_utils.py b/cortex_common/utils/dataframe_utils.py new file mode 100644 index 0000000..10698d1 --- /dev/null +++ b/cortex_common/utils/dataframe_utils.py @@ -0,0 +1,357 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import json +from typing import List, TypeVar, Tuple, Type, Callable + +import arrow +import attr +import pandas as pd + +from cortex_common.utils.object_utils import tuples_with_nans_to_tuples_with_nones, head +from cortex_common.utils.time_utils import seconds_between_times +from cortex_common.utils.type_utils import pass_through_converter + +T = TypeVar("T") +# pylint: disable=line-too-long +# pylint: disable=invalid-name +# pylint: disable=redefined-argument-from-local +# pylint: disable=unnecessary-lambda +# pylint: disable=inconsistent-return-statements +# pylint: disable=bad-mcs-method-argument +# pylint: disable=no-value-for-parameter +# pylint: disable=unnecessary-pass +# pylint: disable=too-few-public-methods + +__all__ = [ + "apply_filter_to_df", + "list_of_attrs_to_df", + "map_column", + "append_seconds_to_df", + "split_df_into_files_based_on_date", + "explode_column", + "filter_time_column_after", + "filter_time_column_before", + "filter_recent_records_on_column", + "parse_set_notation", + "parse_string_set_notation", + "parse_set_of_json_strings_notation", + "head_as_dict", + "df_to_records", + "df_to_tuples", + "df_to_typed_list", + "merge_list_of_dfs_similarly", + "determine_count_of_occurrences_of_grouping", + "determine_time_spent_on_occurrences_of_grouping", +] + + +def apply_filter_to_df(df, filter_lambda): + """ + Functionally apply a filter/mask to a df + :param df: + :param filter_lambda: + :return: + """ + return df[filter_lambda(df)] + + +def list_of_attrs_to_df(l: List) -> pd.DataFrame: + """ + Turns a list of attr based classes into a dataframe + + :param l: + :return: + """ + return pd.DataFrame([attr.asdict(x) for x in l]) + + +def map_column(column: pd.Series, mapper: Callable) -> pd.Series: + """ + Functionally map a column in a df + :param column: + :param mapper: + :return: + """ + return column.map(mapper) + + +def append_seconds_to_df( + df: pd.DataFrame, column_name_to_append: str, start_time_col: str, end_time_col: str +) -> pd.DataFrame: + """ + Appends an additional column that representes the duration between two other time oriented columns within the df. + :param df: + :param column_name_to_append: + :param start_time_col: + :param end_time_col: + :return: + """ + return df.assign( + **{ + column_name_to_append: list( + map( + lambda x: seconds_between_times(arrow.get(x[0]), arrow.get(x[1])), + df[[start_time_col, end_time_col]].itertuples( + index=False, name=None + ), + ) + ) + } + ) + + +def split_df_into_files_based_on_date( + df: pd.DataFrame, on_date: str, file_pattern: str +) -> None: + """ + Saves a dataframe into multiple files ... based on a date column. + Records that occure on the same day with regards to the date column are saved into the same file . + + :param df: Dataframe to split + :param on_date: Date column to split on + :param file_pattern: The pattern to save the new files created, where {date} will be replaced with the actual date ... + :return: Nothing, this function creates new files ... + """ + for date, df_on_date in df.groupby(on_date): + df_on_date.reset_index().to_csv( + file_pattern.format(date=str(arrow.get(date).date())) + ) + + +def explode_column(unindexed_df: pd.DataFrame, column: str) -> pd.DataFrame: + """ + Expands a column where the value of each cell within a column is a list ... + >>> len(explode_column(pd.DataFrame([{"name":"bob", "knows": ["jane", "jack"]}]))["knows"].unique()) + 2 + Assumption: + - df has no index ... + :param df: + :param column: + :return: + """ + if unindexed_df.empty: + return unindexed_df + id_columns = list(set(unindexed_df.columns).difference(set([column]))) + df = ( + (unindexed_df.set_index(id_columns))[column] + .apply(pd.Series) + .stack() + .to_frame(column) + ) + for column in id_columns: + df = df.reset_index(level=column) + return df.reset_index(drop=True) + + +# ------------------ Filtering ------------------------------------------------------------ + + +def filter_time_column_after( + df: pd.DataFrame, time_column: str, shifter: dict +) -> pd.DataFrame: + """ + Filters a dataframes rows based on the date in a specific column ... + + :param df: The Dataframe to filter + :param time_column: The name of the time column to filter + :param shifter: Arrow friendly dict to shift an arrow time ... + :return: + """ + return df[ + df[time_column].map(arrow.get) >= arrow.utcnow().shift(**shifter) + ].reset_index(drop=True) + + +def filter_time_column_before( + df: pd.DataFrame, time_column: str, shifter: dict +) -> pd.DataFrame: + """ + Filters a dataframes rows based on the date in a specific column ... + + :param df: The Dataframe to filter + :param time_column: The name of the time column to filter + :param shifter: Arrow friendly dict to shift an arrow time ... + :return: + """ + return df[ + df[time_column].map(arrow.get) <= arrow.utcnow().shift(**shifter) + ].reset_index(drop=True) + + +def filter_recent_records_on_column( + df: pd.DataFrame, column: str, days_considered_recent +) -> pd.DataFrame: + """ + Filter a df based on recent records in a time based column ... + :param df: + :param column: + :param days_considered_recent: + :return: + """ + return filter_time_column_after(df, column, {"days": -1 * days_considered_recent}) + + +# ------------------ + + +def parse_set_notation(string_series: pd.Series) -> pd.Series: + """ + Turns a series into strings into a series of sets ... + + :param string_series: + :return: + """ + if string_series.empty: + return pd.Series([]) + return string_series.map(parse_string_set_notation) + + +def parse_string_set_notation(string: str) -> pd.Series: + """ + Splits comman seperated string into a pandas series ... + + :param string: + :return: + """ + return set(string[1:-1].split(",")) + + +def parse_set_of_json_strings_notation(string_series: pd.Series) -> pd.Series: + """ + Turns a series of strings into a series of json objects ... + :param string_series: + :return: + """ + if string_series.empty: + return [] + return string_series.map( + pass_through_converter( + (str,), + lambda string: list( + map(lambda x: json.loads(x), json.loads("[{}]".format(string[1:-1]))) + ), + ) + ) + + +# ---------- DF Conversions ---------- + + +def head_as_dict(df: pd.DataFrame) -> dict: + """ + Returns the head of the dataframe into a dict ... + :param df: + :return: + """ + if df.empty: + return {} + dict_as_head = head(df.head(1).to_dict(orient="records")) + return dict_as_head if dict_as_head is not None else {} + + +def df_to_records(df: pd.DataFrame) -> List[dict]: + """ + Turns a Dataframe into a list of dicts ... + :param df: + :return: + """ + # return df.to_dict(orient="records") + return df.to_dict("records") + + +def df_to_tuples(df: pd.DataFrame, columns: List[str]) -> List[Tuple]: + """ + Turns a dataframe into a list of tuples ... + :param df: + :param columns: Names of columns to keep as tuples in order ... + :return: + """ + return tuples_with_nans_to_tuples_with_nones( + df[columns].itertuples(index=False, name=None) + ) + + +def df_to_typed_list(df: pd.DataFrame, t: Type[T]) -> List[T]: + """ + Turns a dataframe into a list of a specific type ... + :param df: + :param t: + :return: + """ + return list( + map( + lambda rec: t(**rec), # type: ignore # not a good way to do this with attr ... + df_to_records(df), + ) + ) + + +# ---------- DF Concatenation ---------- + + +def merge_list_of_dfs_similarly( + group_list: List[pd.DataFrame], **kwargs +) -> pd.DataFrame: + """ + Merged a list of Dataframes ... into a single Dataframe ... on the same criteria ... + :param group_list: + :param kwargs: + :return: + """ + if len(group_list) == 0: + return pd.DataFrame(columns=[kwargs.get("on", kwargs.get("left_on"))]) + if len(group_list) == 1: + return group_list[0] + if len(group_list) == 2: + return pd.merge(group_list[0], group_list[1], **kwargs) + if len(group_list) >= 3: + return merge_list_of_dfs_similarly( + [merge_list_of_dfs_similarly(group_list[:2], **kwargs)] + group_list[2:] + ) + + +# -------------- DF Grouping Utils -------------------------- + + +def determine_count_of_occurrences_of_grouping( + df: pd.DataFrame, grouping: List[str], count_column_name: str = "total" +) -> pd.DataFrame: + """ + Aggregate groupings based on counts of a specific column ... + :param df: + :param grouping: + :param count_column_name: + :return: + """ + return ( + df.groupby(grouping).size().reset_index().rename(columns={0: count_column_name}) + if not df.empty + else pd.DataFrame(columns=grouping + [count_column_name]) + ) + + +def determine_time_spent_on_occurrences_of_grouping( + df: pd.DataFrame, grouping: List[str], time_duration_col: str +) -> pd.DataFrame: + """ + Aggregate a time column representing duration ... + :param df: + :param grouping: + :param time_duration_col: + :return: + """ + return df[grouping + [time_duration_col]].groupby(grouping).sum().reset_index() diff --git a/cortex_common/utils/equality_utils.py b/cortex_common/utils/equality_utils.py new file mode 100644 index 0000000..ca845da --- /dev/null +++ b/cortex_common/utils/equality_utils.py @@ -0,0 +1,129 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import List, Mapping + +from cortex_common.utils.object_utils import head, tail + +# pylint: disable=line-too-long +# pylint: disable=invalid-name +# pylint: disable=use-a-generator +# pylint: disable=unnecessary-lambda + +__all__ = [ + "equal", + "lists_are_equal", + "dicts_are_equal", + "merge_hashes", + "hasher", + "list_hasher", + "dict_hasher", +] + + +def equal(x, y, equality_function=lambda a, b: a == b): + """ + Determines if two things are equal + :param x: + :param y: + :param equality_function: + :return: + """ + if isinstance(x, list): + return lists_are_equal(x, y) + if isinstance(x, dict): + return dicts_are_equal(x, y) + return equality_function(x, y) + + +def lists_are_equal(l1: List, l2: List): + """ + Determines if two lists are equal + :param l1: + :param l2: + :return: + """ + return ( + isinstance(l1, list) + and isinstance(l2, list) + and (len(l1) == len(l2)) + and (all([t1 == t2 for t1, t2 in zip(map(type, l1), map(type, l2))])) + and (all([equal(x, y) for x, y in zip(l1, l2)])) + ) + + +def dicts_are_equal(d1, d2): + """ + Determines if two dicts are equal + :param d1: + :param d2: + :return: + """ + return ( + isinstance(d1, dict) + and isinstance(d2, dict) + and lists_are_equal(sorted(d1.keys()), sorted(d2.keys())) + and all([equal(d1[x], d2[x]) for x in d1.keys()]) + ) + + +def merge_hashes(*hn): + """ + For combining hashes: https://stackoverflow.com/questions/29435556/how-to-combine-hash-codes-in-in-python3 + :return: + """ # pylint: disable=line-too-long + only_1_hash = head(tail(hn)) is None + h1 = head(hn) + h2 = head(tail(hn)) + return ( + hash(None) + if not hn + else (h1 if only_1_hash else merge_hashes(h1 ^ h2, *tail(tail(hn)))) + ) + + +def hasher(x, hash_function=lambda a: hash(a)): + """ + Hashes things in unison with their equality function. + :param x: + :param hash_function: + :return: + """ + if isinstance(x, list): + return list_hasher(x) + if isinstance(x, dict): + return dict_hasher(x) + return hash_function(x) + + +def list_hasher(l1: List): + """ + Hashes a list in unison with its equality function. + :param l1: + :return: + """ + return merge_hashes(*[hasher(x) for x in l1]) + + +def dict_hasher(d1: Mapping): + """ + Hashes a dict in unison with its equality function. + :param d1: + :return: + """ + return merge_hashes( + list_hasher(sorted(d1.keys())), list_hasher([d1[k] for k in sorted(d1.keys())]) + ) diff --git a/cortex_common/utils/etl_utils.py b/cortex_common/utils/etl_utils.py new file mode 100644 index 0000000..729fa82 --- /dev/null +++ b/cortex_common/utils/etl_utils.py @@ -0,0 +1,15 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/cortex_common/utils/generator_utils.py b/cortex_common/utils/generator_utils.py new file mode 100644 index 0000000..6c05993 --- /dev/null +++ b/cortex_common/utils/generator_utils.py @@ -0,0 +1,126 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import itertools +import re +from typing import Callable, Any, List, Optional, Generator + +from cortex_common.utils.object_utils import assign_to_dict +from cortex_common.utils.string_utils import split_string_into_parts + +ToYeild = Any + +__all__ = [ + "get_until", + "get_unique_cortex_objects", + "label_generator", + "chunk_iterable", +] + + +def chunk_iterable(iterable, size) -> Generator[List, None, None]: + """ + From: https://alexwlchan.net/2018/12/iterating-in-fixed-size-chunks/ + :param iterable: + :param size: + :return: + """ + itera = iter(iterable) + while True: + chunk = list(itertools.islice(itera, size)) + if not chunk: + break + yield chunk + + +def get_until( + yielder: Callable[[], Any], + appender: Callable[[Any, ToYeild], Any], + ignore_condition: Callable[[Any, ToYeild], bool], + stop_condition: Callable[[ToYeild], bool], + to_yield: ToYeild, +) -> ToYeild: + """ + Keep on yeilding items from a generator until certain conditions are met, optionally ignoring some generated items + too ... + :param yielder: + :param appender: + :param ignore_condition: + :param stop_condition: + :param to_yield: + :return: + """ # pylint: disable=line-too-long + ignored = 0 + return_val = to_yield + while not stop_condition(return_val): + next_item = yielder() + if ignore_condition(next_item, return_val): + ignored += 1 + else: + return_val = appender(next_item, return_val) + # print(ignored, len(returnVal)) + return return_val + + +def get_unique_cortex_objects(yielder, limit: int) -> List: + """ + Generate unique concepts similar to the Cortex Concept Synthesizor / Provider + :param yielder: + :param limit: + :return: + """ + return list( + get_until( + yielder, + appender=lambda obj, dictionary: assign_to_dict(dictionary, obj["id"], obj), + ignore_condition=lambda obj, dictionary: obj["id"] in dictionary, + stop_condition=lambda dictionary: len(dictionary) >= limit, + to_yield={}, + ).values() + ) + + +def label_generator( + word: str, used_labels: List[str], label_length: int = 3 +) -> Optional[str]: + """ + Right now, labels are only three letters long! + :param word: + :param used_labels: + :return: + """ + words = re.split(r"[^a-zA-Z0-9]", word) + if len(words) != label_length: + word = "".join(words) + words = split_string_into_parts(word, label_length) + try: + return "".join( + next( + filter( + lambda x: "".join(x).upper() not in used_labels, + itertools.product(*words), + ) + ) + ).upper() + except StopIteration: + print("Failed to generate label") + return None + + # longest_word = max(map(len, words)) + # extended_words = [ + # list(word) + ['']*(longest_word-len(word)) for word in words + # ] + # list(itertools.combinations((itertools.chain(*list(zip(*extended_words)))), 3)) diff --git a/cortex_common/utils/id_utils.py b/cortex_common/utils/id_utils.py new file mode 100644 index 0000000..db82664 --- /dev/null +++ b/cortex_common/utils/id_utils.py @@ -0,0 +1,30 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import hashlib + +__all__ = [ + "hash_query", +] + + +def hash_query(query): + """ + Has a string query ... + :param query: + :return: + """ + return hashlib.md5("".join(query.lower().split()).encode("utf-8")).hexdigest() diff --git a/cortex_common/utils/object_utils.py b/cortex_common/utils/object_utils.py new file mode 100644 index 0000000..ba153d4 --- /dev/null +++ b/cortex_common/utils/object_utils.py @@ -0,0 +1,587 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import copy +import itertools +import uuid +from collections import defaultdict +from enum import Enum +from typing import ( + List, + Optional, + Any, + Tuple, + Callable, + Union, + Mapping, + Iterable, + Set, + cast, + TypeVar, + Iterator, +) + +import pandas as pd +import pydash + +from cortex.utils import get_logger + +log = get_logger(__name__) +# pylint: disable=line-too-long +# pylint: disable=invalid-name +# pylint: disable=unnecessary-comprehension +# pylint: disable=consider-using-generator +# pylint: disable=dangerous-default-value +# pylint: disable=cell-var-from-loop +# pylint: disable=unnecessary-lambda +# pylint: disable=logging-fstring-interpolation +# pylint: disable=no-self-argument + +__all__ = [ + "unique_id", + "head", + "tail", + "flatten_list_recursively", + "flatmap", + "append_to_list", + "partition_list", + "dervie_set_by_element_id", + "filter_empty_records", + "search_for_values_in_list", + "nan_to_none", + "tuples_with_nans_to_tuples_with_nones", + "split_list_of_tuples", + "merge_dicts", + "assign_to_dict", + "map_object", + "map_dict_keys", + "nest_values_under", + "append_key_to_values_as", + "drop_from_dict", + "join_inner_arrays", + "invert_dict_lookup", + "pluck", + "group_by_key", + "group_objects_by", + "reverse_index_dictionary", + "map_key", + "merge_unique_objects_on", + "convert_to_string_dict", + "negate_dict_of_numbers", + "merge_enum_values", + "EnumWithCamelCasedNamesAsDefaultValue", + "EnumWithNamesAsDefaultValue", + "rename", + "unzip", + "prune_and_log_nulls_in_list", +] + + +def unique_id() -> str: + """ + Returns a unique id. + """ + return str(uuid.uuid4()) + + +# ------------------------------------------- List Utils ------------------------------------------------ + + +def head(l: Optional[List[Any]]) -> Optional[Any]: + """ + Gets the head of a list if its not empty. + If its empty, None is returned ... + :param l: + :return: + """ + if l is None: + return None + list_with_first_elem = l[0:1] + return list_with_first_elem[0] if list_with_first_elem else None + + +def tail(l: List) -> List: + """ + Returns every element except the first in a list ... + :param l: + :return: + """ + return None if l is None else l[1:] + + +def flatten_list_recursively(l: Union[List[Any], Any], remove_empty_lists=False): + """ + Recursively flattens a list of objects. + :param l: + :param remove_empty_lists: + :return: + """ + if l is None: + return [] + if not isinstance(l, list): + return [l] + # THIS DOES WEIRD STUFF WITH TUPLES! + return list( + itertools.chain( + *[flatten_list_recursively(x) for x in l if (not remove_empty_lists or x)] + ) + ) + + +def flatmap(listToItterate: List, inputToAppendTo: List, function: Callable) -> List: + """ + flatmap impl + :param listToItterate: + :param inputToAppendTo: + :param function: + :return: + """ + if not listToItterate: + return [] + head_attr = listToItterate[0] + tail_attr = listToItterate[1:] + return flatmap(tail_attr, function(inputToAppendTo, head_attr), function) + + +def append_to_list(l: List, thing_to_append: Optional[object]) -> List: + """ + Appends item to a list functionally + :param l: + :param thing_to_append: + :return: + """ + return l + [thing_to_append] if thing_to_append else l + + +def partition_list(l: List, n_partitions: int) -> List[List]: + """ + Splits a list into X different lists + :param l: + :param n_partitions: + :return: + """ + assert n_partitions >= 1, "Partitions must be >= 1" + size_of_each_partition = int(len(l) / n_partitions) + partitions = cast( + List[Tuple[Optional[int], ...]], + zip( + [x for x in range(0, n_partitions)], + [x for x in range(1, n_partitions)] + [None], # type: ignore + ), + ) + return [ + l[ + cast(int, start) + * size_of_each_partition : cast( # type: ignore + Optional[int], (None if end is None else end * size_of_each_partition) + ) # type: ignore + ] + for start, end in partitions # type: ignore + ] + + +def dervie_set_by_element_id( + l: List[Any], identifier: Callable[[Any], str] = lambda x: x +) -> Set[Any]: + """ + Makes a set out of a list, dropping dupplicates, based on a function to determine the id of each element + :param l: + :param identifier: + :return: + """ + # from itertools import combinations + # combinations(l, 2) + return set({identifier(x): x for x in l}.values()) + + +def filter_empty_records(l: List) -> List: + """ + Remove empty records from a list + :param l: + :return: + """ + return [x for x in l if x] + + +def search_for_values_in_list(list_to_search: Iterable, search_query: Callable) -> List: + """ + Returns a subset of the original list based on the elements that match the search function. + :param list_to_search: + :param search_query: + :return: + """ + return list(filter(search_query, list_to_search)) + + +# ------------------------------------------- Tuple Utils ------------------------------------------------ + + +def nan_to_none(value: Any) -> Optional[Any]: + """ + Turns NaNs to Nones ... + :return: + """ + return None if isinstance(value, float) and pd.isna(value) else value + + +def tuples_with_nans_to_tuples_with_nones( + iterator: List[Tuple[Any, ...]] +) -> List[Tuple[Optional[Any], ...]]: + """ + Replaces NaNs within a tuple into Nones. + + # ... I only want to check for NaNs on primitives... and replace them with None ... not Lists ... + # Unfortunately python has no way of saying "isPrimitive" + # Luckily, NaNs are floats ...! + + :param iterator: + :return: + """ + return [tuple([nan_to_none(x) for x in tup]) for tup in iterator] + + +def split_list_of_tuples(l: List[Tuple[Any, ...]]) -> Optional[Tuple[List[Any], ...]]: + """ + NOTE: No python way of specifying that the return type ... the number of List[Any] in it depends on the size of the tuple passed in ... + :param l: + :return: + """ + if not l: + return None + lengths_of_each_tuple = list(map(lambda x: len(list(x)), l)) + # We know there is at least one item ... + all_tuples_same_length = all( + map(lambda x: x == lengths_of_each_tuple[0], lengths_of_each_tuple) + ) + assert all_tuples_same_length, "All tuples must be of the same length: {}".format( + lengths_of_each_tuple[0] + ) + return tuple(*[[tupe[i] for tupe in l] for i in range(0, lengths_of_each_tuple[0])]) + + +# ------------------------------------------- Object Utils ------------------------------------------------ + + +def merge_dicts(a: dict, b: dict) -> dict: + """ + Merges two dicts functionally + :param a: + :param b: + :return: + """ + c = copy.deepcopy(a) + c.update(b) + return c + + +def assign_to_dict(dictionary: dict, key: str, value: object) -> dict: + """ + Assigns an additional k-v pair to a dict functionally + :param dictionary: + :param key: + :param value: + :return: + """ + return merge_dicts(dictionary, {key: value}) + + +def map_object( + obj: Optional[Any], method: Callable[[Any], Any], default: Optional[Any] = None +) -> Any: + """ + Maps an object to another object functionally ... + :param obj: + :param method: + :param default: + :return: + """ + return method(obj) if obj is not None else default + + +def map_dict_keys( + d: Mapping[str, Any], key_mappers: Optional[Mapping[str, Callable]] = None +) -> dict: + """ + Remaps values in dicts based on a collection of mapping functions for the different keys in the dictionary + :param d: + :param key_mappers: + :return: + """ + key_mappers = key_mappers if key_mappers is not None else {} + return {k: v if k not in key_mappers else key_mappers[k](v) for k, v in d.items()} + + +def nest_values_under(d: dict, under: str) -> dict: + """ + Creates a new root key for the dictionary, nesting the passed dictionary under the passed key. + :param d: + :param under: + :return: + """ + return {k: {under: v} for k, v in d.items()} + + +def append_key_to_values_as(d: dict, key_title: str) -> List[dict]: + """ + Appends an array of k-v pairs in a dictionaty as k-v pairs to the original dict ... + :param d: + :param key_title: + :return: + """ + return [pydash.merge(value, {key_title: key}) for key, value in d.items()] + + +def _drop_from_dict(d: dict, skip: List[object]) -> dict: + """ + Internal method to help drop elements from a dict. + :param d: + :param skip: + :return: + """ + if d is None: + d = None + if isinstance(d, list): + return [drop_from_dict(e, skip) for e in d] + if isinstance(d, dict): + return {k: drop_from_dict(v, skip) for k, v in d.items() if k not in skip} + return d + + +def drop_from_dict(d: dict, skip: List[object]) -> dict: + """ + Drop any specified elements from the dict recursively + :param d: + :param skip: + :return: + """ + return _drop_from_dict(d, skip) + + +def join_inner_arrays(_dict: dict, caster=lambda x: x) -> dict: + """ + For any arrays in the dictionary, join its elements into a comma seperated string, optionally casting each element + pre - join. + :param _dict: + :param caster: + :return: + """ + return { + k: ",".join(map(caster, v)) if isinstance(v, list) else v + for (k, v) in _dict.items() + } + + +def invert_dict_lookup(d: dict) -> dict: + """ + Flip the key-value pairs in a dict. + :param d: + :return: + """ + return {v: k for k, v in d.items()} + + +def pluck(path, d, default={}): + """ + Get a sepecific element from a dict. + :param path: + :param d: + :param default: + :return: + """ + split_path = [x for x in path.split(".") if x] + if len(split_path) > 0: + return pluck(".".join(split_path[1:]), d.get(split_path[0], default)) + return d + + +def group_by_key(l: List[Any], key: Callable[[Any], str]) -> Mapping[str, List[Any]]: + """ + Create groups of elements from a list, based on a grouping function. + :param l: + :param key: + :return: + """ + key_deriver = key if callable(key) else lambda x: x[key] + returnVal: Mapping[str, List] = defaultdict(list) + for x in l: + returnVal[key_deriver(x)].append(x) + return returnVal + + +def group_objects_by( + l: List[Any], group_by: Callable[[Any], str] +) -> Mapping[str, List[Any]]: + """ + Create groups of elements from a list, based on a grouping function. + TODO whats the diff between this and group_by_key + :param l: + :param group_by: + :return: + """ + unique_groups = set(map(group_by, l)) + return {g: list(filter(lambda x: group_by(x) == g, l)) for g in unique_groups} + + +def reverse_index_dictionary(d: dict) -> dict: + """ + Invert keys and values of a dictionary, hanging onto any duplicate values that occure pre-reversing + :param d: + :return: + """ + new_keys = list(set(flatten_list_recursively(list(d.values())))) + return { + new_key: [old_key for old_key in list(d.keys()) if new_key in d[old_key]] + for new_key in new_keys + } + + +def map_key(o: dict, key: str, mapper: Callable) -> dict: + """ + Map the value of a specific key in a dictionary. + :param o: + :param key: + :param mapper: + :return: + """ + return pydash.set_(o, key, mapper(pydash.get(o, key))) + + +def merge_unique_objects_on( + objects: List[Any], identifier: Callable, reducer: Callable = head +) -> List[Any]: + """ + Reduce a list of objects that are in the same group (determined by a grouping function) based on the + reduction function. + :param objects: + :param identifier: + :param reducer: + :return: + """ + groups = group_by_key(objects, identifier) + return list( + {groupId: reducer(values) for groupId, values in groups.items()}.values() + ) + + +def convert_to_string_dict(d: dict) -> dict: + """ + Cast keys and values in dict to strings. + :param d: + :return: + """ + return {str(k): str(v) for k, v in d.items()} + + +def negate_dict_of_numbers(d: dict) -> dict: + """ + For dicts with number values, invert the numbers. + :param d: + :return: + """ + return {k: -1 * v for k, v in d.items()} + + +# ------------------------------------------ ENUM Utils ------------------------------------------ + + +class EnumWithCamelCasedNamesAsDefaultValue(Enum): + """ + Enum where auto values are CamelCased + """ + + def _generate_next_value_(name, start, count, last_values): + return pydash.strings.camel_case(name) + + +class EnumWithNamesAsDefaultValue(Enum): + """ + Enum where auto values are the raw name of the auto value. + """ + + def _generate_next_value_(name, start, count, last_values): + return name + + +def merge_enum_values( + values: List[Enum], + merger: Callable[[list], object] = lambda values: ".".join(values), +) -> object: + """ + Merges enum values + :param values: + :param merger: + :return: + """ + return merger(list(map(lambda x: x.value, values))) + + +def unzip(list_of_tuples: List) -> List[List]: + """ + Turns a list of tuples into a list of lists ... where the first list groups all the first elements of the tuples ... + + Unzips a list of tuples into a list of the first elements, second elements, ... + Tuples all have to be of equal length ... + + :param list_of_tuples: + :return: + """ + return list(map(list, zip(*list_of_tuples))) + + +def rename(d: dict, keys_to_rename: List[Tuple[str, str]]) -> dict: + """ + Capable of renaming the same key into multiple new keys ... + :param d: + :param keys_to_rename: + :return: + """ + if not keys_to_rename: + return d + + # keys_that_will_be_renamed = unzip(keys_to_rename)[0] + keys_to_keep = pydash.omit( + d, unzip(keys_to_rename)[0] + ) # Omit all keys to be renamed ... + renamed_keys = [ + pydash.rename_keys(pydash.pick(d, list(renamer.keys())[0]), renamer) + for renamer in map(lambda pair: dict([pair]), keys_to_rename) + if list(renamer.keys())[0] + in d # Omit all renaming for keys that dont exist in the dict to be renamed ... + ] + post_rename = pydash.merge({}, keys_to_keep, *renamed_keys) + # print(f"keeping everything other than {keys_that_will_be_renamed} from {d} : {keys_to_keep}") + # print(f"post_rename: {post_rename}") + # print(f"renamed_keys: {renamed_keys}") + return post_rename + + +ValidT = TypeVar("ValidT") + + +def prune_and_log_nulls_in_list( + l: Iterable[Optional[ValidT]], iterable_label: str = "iterable" +) -> Iterator[ValidT]: + """ + Logs and removes Nones from a list ... + :param l: + :param iterable_label: + :return: + """ + for i, item in enumerate(l): + if item is None: + log.warning(f"Pruned None value at index {i} within {iterable_label}.") + else: + yield item diff --git a/cortex_common/utils/string_utils.py b/cortex_common/utils/string_utils.py new file mode 100644 index 0000000..4f49990 --- /dev/null +++ b/cortex_common/utils/string_utils.py @@ -0,0 +1,74 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import itertools +import re +from typing import List, cast + +from cortex_common.utils.object_utils import head, tail + +# pylint: disable=line-too-long +# pylint: disable=invalid-name +# pylint: disable=no-else-return + +__all__ = [ + "split_camel_case", + "split_string_into_parts", +] + + +def split_camel_case(string: str) -> List[str]: + """ + Turns "CLevelChangeInsights" into ['C', 'Level', 'Change ', 'Insights'] + :param string: + :return: + """ + l = [x for x in re.split(r"([A-Z])", string) if x] + if not l: + return l + if not tail(l): + return l + + upper_case_chrs = list(map(chr, range(ord("A"), ord("Z") + 1))) + lower_case_chrs = list(map(chr, range(ord("a"), ord("z") + 1))) + if head(l) in upper_case_chrs and head(head(tail(l))) in upper_case_chrs: + return cast(List[str], [head(l)]) + split_camel_case("".join(tail(l))) + elif head(l) in upper_case_chrs and head(head(tail(l))) in lower_case_chrs: + return cast( + List[str], ["{}{}".format(head(l), head(tail(l)))] + ) + split_camel_case("".join(tail(tail(l)))) + else: + return cast(List[str], [head(l), head(tail(l))]) + split_camel_case( + "".join(tail(tail(l))) + ) + + +def split_string_into_parts(string: str, num_of_parts: int) -> List: + """ + Split up a string ... + :param string: + :param num_of_parts: + :return: + """ + l = len(string) + splittings = list( + zip( + [0] + + list(map(lambda x: int(x * l / num_of_parts), range(1, num_of_parts))), + list(map(lambda x: int(x * l / num_of_parts), range(1, num_of_parts))) + [None], # type: ignore + ) + ) + return ["".join(list(itertools.islice(string, x, y))) for x, y in splittings] diff --git a/cortex_common/utils/time_utils.py b/cortex_common/utils/time_utils.py new file mode 100644 index 0000000..c2fea7b --- /dev/null +++ b/cortex_common/utils/time_utils.py @@ -0,0 +1,265 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from functools import wraps +import sys +import time +from typing import Any, Mapping, Iterable, Tuple, Dict + +import arrow +import pandas as pd + +from cortex_common.utils.object_utils import negate_dict_of_numbers + +# pylint: disable=chained-comparison +# pylint: disable=broad-exception-caught +# pylint: disable=invalid-name + + +__all__ = [ + "determine_weekly_ranges", + "utc_timestamp", + "timeit", + "timeit_safely", + "derive_hour_from_date", + "derive_day_from_date", + "remap_date_formats", + "seconds_between_times", + "fold_start_and_stop_time_tuples_into_dict", + "oldest", + "newest", + "time_is_within", +] + + +def utc_timestamp() -> str: + """ + Gets an ISO-8601 complient timestamp of the current UTC time. + :return: + """ + return str(arrow.utcnow()) + + +def timeit(method): + """ + A decorator that times the invocation of a method and returns it along with the response from the method as a tuple. + :param method: + :return: + """ # pylint: disable=line-too-long + + @wraps(method) + def timed(*args, **kw): + t_s = time.time() + result = method(*args, **kw) + t_e = time.time() + return "%2.2f" % (t_e - t_s), result + + return timed + + +def timeit_safely(precision=2): + """ + Wrapper to time a method, even if it throws an exception. + :param precision: + :return: + """ + + def _timeit_safely(method): + """ + A decorator that times the invocation of a method and returns it along with the response from the method as a tuple. + :param method: + :return: + """ # pylint: disable=line-too-long + + @wraps(method) + def timed(*args, **kw): + t_s = time.time() + result = None + exception = None + try: + result = method(*args, **kw) + except Exception: + exception = sys.exc_info() + t_e = time.time() + return (f"%2.{precision}f" % (t_e - t_s), result, exception) + + return timed + + return _timeit_safely + + +def derive_hour_from_date(iso_timestamp: str) -> dict: + """ + Enriches an ISO UTC Timestamp ... + :param iso_timestamp: + :return: + """ + d = arrow.get(iso_timestamp) + return { + "hour_number": int(d.format("H")), + "hour": d.format("hhA"), + "timezone": d.format("ZZ"), + } + + +def derive_day_from_date(iso_timestamp) -> str: + """ + Derives the day from a ISO UTC Timestamp ... + >>> derive_day_from_date('2019-03-27T21:18:21.760245+00:00') == '2019-03-27' + :param iso_timestamp: + :return: + """ + return str(arrow.get(iso_timestamp).date()) + + +def remap_date_formats( + date_dict: Mapping[Any, arrow.Arrow], date_formats, original_format +) -> Mapping[Any, arrow.Arrow]: + """ + Maps a date from on format to another ... + + :param date_dict: + :param date_formats: + :param original_format: + :return: + """ + return { + k: arrow.get(v, original_format).format(date_formats.get(k, original_format)) + for (k, v) in date_dict.items() + } + + +def seconds_between_times( + arrow_time_a: arrow.Arrow, arrow_time_b: arrow.Arrow +) -> float: + """ + Finds the amount of seconds between two arrow times ... + :param arrow_time_a: + :param arrow_time_b: + :return: + """ + return abs(arrow_time_a.float_timestamp - arrow_time_b.float_timestamp) + + +def fold_start_and_stop_time_tuples_into_dict( + startTime_stopTime_tuples: Iterable[Tuple], +) -> Dict: + """ + Helper to figure out max overlapping start and stop times ... + >>> [ + >>> ("2019-01-01T00:00:00Z", "2019-01-01T01:00:00Z"), ("2019-01-01T00:00:00Z", "2019-01-02T02:00:00Z") + >>> ] + >>> # into ... + >>> { "2019-01-01T00:00:00Z": "2019-01-02T02:00:00Z"} + + :param startTime_stopTime_tuples: + :return: + """ # pylint: disable=line-too-long + d: Dict = {} + for start_time, stop_time in startTime_stopTime_tuples: + if start_time in d: + # Take the newer stop time ... + if stop_time > d[start_time]: + d[start_time] = stop_time + else: + d[start_time] = stop_time + return d + + +def oldest(list_of_times: Iterable) -> object: + """ + Find oldest time ... + :param list_of_times: + :return: + """ + if not list_of_times: + return None + return sorted(list_of_times, key=lambda x: x)[0] + + +def newest(list_of_times: Iterable) -> object: + """ + Find newest time .. + :param list_of_times: + :return: + """ + if not list_of_times: + return None + return sorted(list_of_times, key=lambda x: x)[-1] + + +def time_is_within(time_to_check, time_to_shift, time_shifter): + """ + Before and after ... + :param time_to_check: + :param time_to_shift: + :param time_shifter: + :return: + """ + return ( + time_to_check >= time_to_shift.shift(**negate_dict_of_numbers(time_shifter)) + and time_to_check < time_to_shift + ) or ( + time_to_check < time_to_shift.shift(**time_shifter) + and time_to_check >= time_to_shift + ) + + +def determine_weekly_ranges(dates): + """ + For the dates ... determine week ranges that capture all dates ... + Assumption ... first item in range is included, but last is not ... + :param dates: + :return: + """ + first_date = min(dates) + last_date = max(dates) + # Make sure beginning is sunday ... or move it back to last sunday ... + # Move beginning to sunday ... if still same date ... use it ... (already sunday) + # If different date ... go back a week + first_date_already_sunday = ( + first_date + pd.offsets.Week(n=0, weekday=6) + ) == first_date + sunday_of_range_start = ( + first_date + if first_date_already_sunday + else first_date + pd.offsets.Week(n=-1, weekday=6) + ) + last_date_already_sunday = ( + last_date + pd.offsets.Week(n=0, weekday=6) + ) == last_date + sunday_of_range_end = ( + last_date + pd.offsets.Week(n=1, weekday=6) + if last_date_already_sunday + else last_date + pd.offsets.Week(n=0, weekday=6) + ) + result_list = list( + zip( + pd.date_range( + start=sunday_of_range_start, + end=sunday_of_range_end, + freq="W-SUN", + closed=None, + ), + pd.date_range( + start=sunday_of_range_start + pd.offsets.Week(n=1, weekday=6), + end=sunday_of_range_end + pd.offsets.Week(n=1, weekday=6), + freq="W-SUN", + closed=None, + ), + ) + ) + # Drop last item from list + return result_list[0:-1] diff --git a/cortex_common/utils/type_utils.py b/cortex_common/utils/type_utils.py new file mode 100644 index 0000000..04b3c5d --- /dev/null +++ b/cortex_common/utils/type_utils.py @@ -0,0 +1,156 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import typing +from typing import TypeVar, Callable, Any, Tuple, Union, Optional, List + +import numpy as np + +NoneType = type(None) +T = TypeVar("T") + +# pylint: disable=invalid-name +__all__ = [ + "is_typing_type", + "get_types_of_union", + "is_optional_type", + "is_union_type", + "pass_through_converter", + "fold_list", + "numpy_type_to_python_type", + "is_primitive", +] + + +def is_typing_type(t: Any) -> bool: + """ + Is a type from the typing library ... + Confirmed true for the following typing types: + - typing.Mapping + - typing.Tuple + - typing.Callable + - typing.Type + - typing.List + - typing.Dict + - typing.DefaultDict + - typing.Set + - typing.FrozenSet + - typing.Counter + - typing.Deque + :param t: + :return: + """ + return hasattr(t, "__origin__") and t.__origin__ in [typing.Union, typing.Generic] + + +def get_types_of_union(union: Any) -> Tuple: + """ + Gets all of the types associated with the union type ... + :param union: + :return: + """ + return tuple(union.__args__) if hasattr(union, "__args__") else tuple() + + +def is_optional_type(t: Any) -> bool: + """ + Determines if a type is an Option Type ... + :param t: + :return: + """ + # return is_typing_type(t) and len(t.__args__) == 2 and type(None) in t.__args__ + # return t.__origin__ in [typing.Optional] + return repr(t).startswith("typing.Optional") + + +def is_union_type(t: type) -> bool: + """ + Determines if a type is a Union Type ... + :param t: + :return: + """ + # return t.__origin__ in [typing.Union] + return repr(t).startswith("typing.Union") + + +def pass_through_converter( + types_that_need_conversion: Tuple[Any, ...], converter_method: Callable +) -> Callable[[Any], Any]: + """ + Returns a method that when invoked with the values, determines whether or not the value should be converted. + Items that don't need conversion are passed through as is ... + + :param types_that_need_conversion: + :param converter_method: + :return: + """ # pylint: disable=line-too-long + return ( + lambda x: converter_method(x) + if isinstance(x, types_that_need_conversion) + else x + ) + + +# pylint: disable=line-too-long +# Union feels like a hack ... you can't really treat it as a type ... you almost have to explicitly fill in the union types every time ... +# def union_type_from_tuples(tup:Tuple[Any,...]) -> Union: +# return Union[tup] + +OptionalList = Union[ + Optional[List[Optional[Any]]], + Optional[List[Any]], + List[Any], + List[Optional[Any]], +] + + +def fold_list(l: OptionalList) -> List[Any]: + """ + Folds a list that can optionally contain items by removing Null items ... + :param l: + :return: + """ + if l is None: + return [] + return [x for x in l if x] + + +def numpy_type_to_python_type(value): + """ + Turns numpy types into python types ... + :param value: + :return: + """ + return ( + int(value) + if isinstance(value, (int, np.integer)) + else (float(value) if isinstance(value, (float, np.floating)) else value) + ) + + +def is_primitive(value: Union[int, float, str, bool, None, Any]) -> bool: + """ + Tests if a value is primitive + :param value: + :return: + """ + if isinstance(value, (float, int)): + return True + if isinstance(value, str): + return True + if isinstance(value, bool): + return True + return False diff --git a/cortex_common/validators/__init__.py b/cortex_common/validators/__init__.py new file mode 100644 index 0000000..a5310a2 --- /dev/null +++ b/cortex_common/validators/__init__.py @@ -0,0 +1,17 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from .lists import ListItemsAreInstanceOf diff --git a/cortex_common/validators/lists.py b/cortex_common/validators/lists.py new file mode 100644 index 0000000..7948707 --- /dev/null +++ b/cortex_common/validators/lists.py @@ -0,0 +1,63 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import List +from attr import attrs, attrib + + +@attrs(repr=False, slots=True, hash=True) +class ListItemsAreInstanceOf: + """ + Checks if items of a list are the right value ... + """ + + type = attrib() + nullable = attrib(type=bool, default=False) # Is the list itself nullable? + + def __call__(self, inst, attr, value: List): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if value is None: + if self.nullable: + pass + else: + raise TypeError("'{name}' can not be None.".format(name=attr.name)) + for item in value: + self.check_if_item_is_instance(attr, item) + + def check_if_item_is_instance(self, attr, value): + """ + check_if_item_is_instance + :param attr: + :type attr: + :param value: + :type value: + :return: + :rtype: + """ # pylint: disable=line-too-long + if not isinstance(value, self.type): + raise TypeError( + "All instances of '{name}' must be {type!r} (got {value!r} that is actually a {actual!r}).".format( + name=attr.name, + type=self.type, + actual=value.__class__, + value=value, + ) + ) + + def __repr__(self): + return "".format(type=self.type) diff --git a/cortex_profiles/__init__.py b/cortex_profiles/__init__.py new file mode 100644 index 0000000..934da86 --- /dev/null +++ b/cortex_profiles/__init__.py @@ -0,0 +1,18 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from cortex_profiles.ext import * +from cortex_profiles.ext.rest import * diff --git a/cortex_profiles/datamodel/__init__.py b/cortex_profiles/datamodel/__init__.py new file mode 100644 index 0000000..729fa82 --- /dev/null +++ b/cortex_profiles/datamodel/__init__.py @@ -0,0 +1,15 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/cortex_profiles/datamodel/constants.py b/cortex_profiles/datamodel/constants.py new file mode 100644 index 0000000..94210da --- /dev/null +++ b/cortex_profiles/datamodel/constants.py @@ -0,0 +1,88 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from cortex_common.constants.contexts import CONTEXTS +from cortex_common.utils.config_utils import AttrsAsDict + +# pylint: disable=too-few-public-methods +# pylint: disable=invalid-name + +__all__ = [ + "TIMEFRAMES", + "PROFILE_TYPES", + "UNIVERSAL_ATTRIBUTES", + "DOMAIN_CONCEPTS", + "INTERACTIONS", +] + + +class TIMEFRAMES(AttrsAsDict): + """ + Built in Time Frames + """ + + HISTORIC = "eternally" + RECENT = "recently" + + +class PROFILE_TYPES(AttrsAsDict): + """ + Built in Profile Types + """ + + INSIGHT_CONSUMER = "profile/insight-consumer" + ENTITY_TAGGED_IN_INSIGHTS = "profile/entity-in-insight" + APP_USER = "profile/app-user" + + +class UNIVERSAL_ATTRIBUTES(AttrsAsDict): + """ + Built in Universal Attributes + """ + + TYPES = "profile.types" + + @staticmethod + def keys(): + """ + keys + :return: + :rtype: + """ + return list(filter(lambda x: x[0] != "_", CONTEXTS.__dict__.keys())) + + +class DOMAIN_CONCEPTS(AttrsAsDict): + """ + Built in Concept Types + """ + + PERSON = "cortex/person" + COUNTRY = "cortex/country" + CURRENCY = "cortex/currency" + COMPANY = "cortex/company" + WEBSITE = "cortex/website" + + +class INTERACTIONS(AttrsAsDict): + """ + Built in Interactions + """ + + CONTEXT = CONTEXTS.INSIGHT_INTERACTION + PRESENTED = "presented" + VIEWED = "viewed" + IGNORED = "ignored" diff --git a/cortex_profiles/datamodel/dataframes.py b/cortex_profiles/datamodel/dataframes.py new file mode 100644 index 0000000..5aea9fc --- /dev/null +++ b/cortex_profiles/datamodel/dataframes.py @@ -0,0 +1,50 @@ +""" +Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +from cortex_common.utils.config_utils import AttrsAsDict + +# - [ ] Function to auto derive df schema from name ... +# - [ ] Detail df schemas - Mark Unique Keys Mark Foreign Keys + +# pylint: disable=invalid-name +# pylint: disable=too-few-public-methods + +__all__ = [ + "TAGGED_CONCEPT", + "INTERACTION_DURATIONS_COLS" +] + + +class TAGGED_CONCEPT(AttrsAsDict): + """ + Schema of DF capturing a tagged concept + """ + + TYPE = "taggedConceptType" + RELATIONSHIP = "taggedConceptRelationship" + ID = "taggedConceptId" + TITLE = "taggedConceptTitle" + TAGGEDON = "taggedOn" + + +class INTERACTION_DURATIONS_COLS(AttrsAsDict): + """ + Columns expected of DFs that capture records that lasted a specific duration. + """ + + STARTED_INTERACTION = "startedInteractionISOUTC" + STOPPED_INTERACTION = "stoppedInteractionISOUTC" diff --git a/cortex_profiles/ext/__init__.py b/cortex_profiles/ext/__init__.py new file mode 100644 index 0000000..c265105 --- /dev/null +++ b/cortex_profiles/ext/__init__.py @@ -0,0 +1,75 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from cortex_profiles.ext import clients +from cortex_profiles.ext.rest import * + +from cortex.utils import get_logger +from cortex.client import Client + +log = get_logger(__name__) + +__all__ = ["ProfileClient"] + +# pylint: disable=invalid-name +# pylint: disable=redefined-outer-name + + +class ProfileClient: + """ + Extends the Client Pattern for Profiles with regards to the cortex-python package. + """ + + def __init__(self, client: Client): + """ + Initialize Instance + :param client: Cortex Client + """ + self.project = client._project + self.profile_client = ProfilesRestClient(client) + + def profile(self, profile_id: str, schema_id: str) -> Optional[clients.Profile]: + """ + Returns the latest version of the profile against a specific schema + :param profile_id: Profile ID + :param schema_id: Profile Schema Name + :return: clients.Profile + """ + return clients.Profile.get_profile(profile_id, schema_id, self.profile_client) + + def profile_schema(self, schema_id: str) -> Optional[clients.ProfileSchema]: + """ + Returns the Latest version of Profile Schema + :param schema_id: Profile Schema Name + :return: clients.ProfileSchema + """ + return clients.ProfileSchema.get_schema(schema_id, self.profile_client) + + +if __name__ == "__main__": + from cortex import Cortex + import json + import attr + + endpoint = "https://api.dci-dev.dev-eks.insights.ai" + token = "" + project = "" + client = Cortex.client(api_endpoint=endpoint, token=token, project=project) + pc = ProfileClient(client=client) + profile = pc.profile("07C68D9A1FC5", "member-stream2-8d02a") + + if profile is not None: + print(json.dumps(attr.asdict(profile.latest()), indent=4)) diff --git a/cortex_profiles/ext/builders.py b/cortex_profiles/ext/builders.py new file mode 100644 index 0000000..c0a73ca --- /dev/null +++ b/cortex_profiles/ext/builders.py @@ -0,0 +1,142 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import json +from typing import List +import attr + +from cortex_common.utils.attr_utils import dict_to_attr_class +from cortex_common.types.schemas import ( + ProfileNames, + DataSourceSelection, + JoinSourceSelection, + ProfileSchema, +) + +# pylint: disable=line-too-long + + +class ProfileSchemaBuilder: + """ + ProfileSchemaBuilder class + """ + @staticmethod + def append_from_schema_json(filepath): + """ + Appends Schema from JSON file + :param filepath: + :return: Profile Schema + """ + with open(filepath, "rb") as file: + schema = json.load(file) + return dict_to_attr_class(schema, ProfileSchema) + + def __init__(self, name: str, schema: ProfileSchema = ProfileSchema()): + """ + Initializes the builder from the profile schema type ... + :param schema: + :return: + """ + self._schema = attr.evolve(schema, name=name) + + def name(self, name: str) -> "ProfileSchemaBuilder": + """ + Sets the name of the schema ... + :param name: + :return: + """ + self._schema = attr.evolve(self._schema, name=name) + return self + + def title(self, title: str) -> "ProfileSchemaBuilder": + """ + Sets the title of the schema ... + :param title: + :return: + """ + self._schema = attr.evolve(self._schema, title=title) + return self + + def description(self, description: str) -> "ProfileSchemaBuilder": + """ + Sets the description of the schema ... + :param description: + :return: + """ + self._schema = attr.evolve(self._schema, description=description) + return self + + def project(self, project: str) -> "ProfileSchemaBuilder": + """ + Sets the project of the schema ... + :param project: + :return: + """ + self._schema = attr.evolve(self._schema, project=project) + return self + + def primary_source( + self, primary_source: DataSourceSelection + ) -> "ProfileSchemaBuilder": + """ + Sets the primary_source of the schema ... + :param primary_source: + :return: + """ + self._schema = attr.evolve(self._schema, primarySource=primary_source) + return self + + def joins(self, joins: List[JoinSourceSelection]) -> "ProfileSchemaBuilder": + """ + Sets the JoinSourceSelections of the schema ... + :param joins: + :return: + """ + self._schema = attr.evolve(self._schema, joins=joins) + return self + + def names(self, names: ProfileNames) -> "ProfileSchemaBuilder": + """ + Sets the names of the schema ... + :param names: + :return: + """ + self._schema = attr.evolve(self._schema, names=names) + return self + + def custom_attributes(self, custom_attributes) -> "ProfileSchemaBuilder": + """ + Sets the Custom Attributes + :param custom_attributes: + :return: + """ + self._schema = attr.evolve(self._schema, customAttributes=custom_attributes) + return self + + def bucket_attributes(self, bucket_attributes: object) -> object: + """ + Sets the Bucketed Attributes + :param bucket_attributes: + :return: + """ + self._schema = attr.evolve(self._schema, bucketAttributes=bucket_attributes) + return self + + def build(self) -> ProfileSchema: + """ + Builds a new Profile Schema using the attributes added to the builder + :return: + """ + return self._schema diff --git a/cortex_profiles/ext/clients.py b/cortex_profiles/ext/clients.py new file mode 100644 index 0000000..c9140d5 --- /dev/null +++ b/cortex_profiles/ext/clients.py @@ -0,0 +1,205 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import warnings +from typing import Optional, Union + +from cortex_profiles.ext.rest import ProfilesRestClient + +from cortex_common.types import ( + ProfileSchema as ProfileSchemaType, + ProfileAttribute as ProfileAttributeType, + Profile as ProfileType, + JobInfo, +) + +from cortex.camel import Document + + + +class ProfileAttribute(Document): + """ + Accessing an existent attribute within a Cortex Profile. + Not meant to be explicitly instantiated by sdk users. + """ + + def __init__(self, params: dict, profile_client: ProfilesRestClient): + super().__init__(params, True) + self._profile_client = profile_client + + @staticmethod + def get_attribute( + profile_id: str, attribute_key: str, profile_client: ProfilesRestClient + ) -> Optional["ProfileAttribute"]: + """ + Fetches a profile adhering to a specific schema ... + :param profile_id: ProfileId + :param attribute_key: Attribute Key of a Profile Attribute + :param profile_client: ProfileRestClient instance + :return: ProfileAttribute + """ + return ProfileAttribute( + { + "attribute_key": attribute_key, + "profile_id": profile_id, + }, + profile_client, + ) + + def latest(self, schema_id: str) -> Optional[ProfileAttributeType]: + """ + Returns the latest version of the profile against a specific schema ... + :param schema_id: Profile Schema Name + :return: ProfileAttribute Instance + """ + return self._profile_client.describe_attribute_by_key( + self.profile_id, schema_id, self.attribute_key + ) + + def __repr__(self): + return "".format( + self.attribute_key, self.profile_id + ) + + +class Profile(Document): + """ + Accessing an existent Cortex Profile. + Not meant to be explicitly instantiated by sdk users. + """ + + def __init__( + self, profile: Union[dict, ProfileType], profile_client: ProfilesRestClient + ): + """ + Initialize Instance + :param profile: ProfileInstance or Dict + :param profile_client: ProfilesRestClient Instance + """ + super().__init__(dict(profile), True) + self._profile_client = profile_client + + @staticmethod + def get_profile( + profile_id: str, schema_id: str, profile_client: ProfilesRestClient + ) -> Optional["Profile"]: + """ + Fetches a profile adhering to a specific schema ... + :param profile_id: Profile Id + :param schema_id: Schema ID + :param profile_client: ProfilesRestClient Instance + :return: Profile Instance + """ + return Profile( + {"profile_id": profile_id, "schema_id": schema_id}, profile_client + ) + + def attribute(self, attribute_key: str) -> Optional[ProfileAttribute]: + """ + Get Profile Attribute Value By Key + :param attribute_key: Attribute Name of the Profile + :return: ProfileAttribute Instance + """ + return ProfileAttribute.get_attribute( + self.profile_id, attribute_key, self._profile_client + ) + + def latest(self) -> Optional[ProfileType]: + """ + Returns the latest version of the profile against a specific schema ... + :return: Profile Instance + """ + return self._profile_client.describe_profile(self.profile_id, self.schema_id) + + def delete(self) -> Optional[JobInfo]: + """ + Deletes the profile built against a specific schema. + :return: JobInfo + """ + if self.schema_id is None: + warnings.warn( + "SchemaId must be provided to delete profile against a specific schema." + ) + return None + return self._profile_client.delete_profile(self.profile_id, self.schema_id) + + def delete_all(self) -> Optional[JobInfo]: + """ + Deletes all instances of the profile against all schemas. + :return: JobInfo + """ + return self._profile_client.delete_all_profiles(profile_schema=self.schema_id) + + def __repr__(self): + return "".format(self.profile_id) + + +class ProfileSchema(Document): + """ + Accessing an existent Cortex ProfileSchema. + """ + + def __init__(self, schema_name_or_id: str, profile_client: ProfilesRestClient): + self._schema_requested = schema_name_or_id + self._profile_client = profile_client + self._refresh_schema() + super().__init__(self._schema.to_dict() if self._exists else {}, True) + + @staticmethod + def get_schema(schema_id, profile_client) -> Optional["ProfileSchema"]: + """ + Fetches the requested schema by id ... + :param schema_id: + :param profile_client: + :return: self + """ + return ProfileSchema(schema_id, profile_client) + + def _refresh_schema(self): + """ + Refreshes the internal state of the schema ... + This is only really needed prior to deleting ... + Or ... prior to getting the latest versions of an updated schema + :return: None + """ + self._schema = self._profile_client.describe_schema(self._schema_requested) + self._exists = self._schema is not None + + def latest(self) -> Optional[ProfileSchemaType]: + """ + Get the Latest Profile Schema + :return: ProfileSchema + """ + self._refresh_schema() + return self._schema + + def delete(self) -> bool: + """ + Delete the Profile Schema + :return: Status(True/False) + """ + return self._profile_client.delete_schema(self._schema_requested) + + def exists(self) -> bool: + """ + Returns whether or not the schema requested actually exists ... + :return: Status(True/False) if a Profile Schema Exists or Not + """ + self._refresh_schema() + return self._exists + + def __repr__(self): + return repr(self._schema) if self._exists else repr(None) diff --git a/cortex_profiles/ext/rest.py b/cortex_profiles/ext/rest.py new file mode 100644 index 0000000..1ca26c8 --- /dev/null +++ b/cortex_profiles/ext/rest.py @@ -0,0 +1,704 @@ +""" +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import json +from typing import List, Optional, Union + +from cortex_common.utils import dicts_to_classes, dict_to_attr_class, head +import cortex_common.constants.queries as gql_queries +from cortex_common.types import ( + ProfileSchema, + Profile, + ProfileFeature, + GroupCount, + ProfileCommit, + ProfileAttribute, + JobInfo, + CustomAttributeSpec, + BucketAttributeSpec, +) + +from cortex.serviceconnector import _Client + +# pylint: disable=broad-exception-caught +# pylint: disable=broad-exception-raised +# pylint: disable=invalid-name +# pylint: disable=inconsistent-return-statements +# pylint: disable=redefined-builtin +# pylint: disable=consider-using-with +# pylint: disable=too-many-public-methods +# pylint: disable=line-too-long + + +class ProfilesRestClient(_Client): + """ + A client used to manage profiles. + """ + + URIs = {"profiles": "graphql"} + + def _query(self, query: str, variables: dict, ip_type: str): + result = None + try: + headers = {"Content-Type": "application/json"} + result = self._serviceconnector.request( + "POST", + uri=self.URIs["profiles"], + body=json.dumps({"variables": variables, "query": query}), + headers=headers, + ) + response = result.json() + if "data" not in response and "errors" in response: + raise Exception(response["errors"]) + return response["data"][ip_type] + except Exception as e: + print(result, e) + + def describe_schema(self, schema_name: str) -> Optional[ProfileSchema]: + """ + Describe Profile Schema + :param schema_name: Profile Schema Name + :return: ProfileSchema + """ + schema = self._query( + gql_queries.PROFILE_SCHEMA_SUMMARY, + variables={"project": self._project(), "name": schema_name}, + ip_type="profileSchemaByName", + ) + return dict_to_attr_class(schema, ProfileSchema) + + def list_schemas(self) -> Optional[List[ProfileSchema]]: + """ + List Profile Schemas in a Project + :return: List[ProfileSchema] + """ + schemas = self._query( + gql_queries.FIND_SCHEMAS, + variables={"project": self._project()}, + ip_type="profileSchemas", + ) + return dicts_to_classes(schemas, ProfileSchema) + + def find_profiles( + self, + profile_schema: str, + attributes: List[str] = None, + filter: str = None, + limit: int = None, + ) -> Optional[List[Profile]]: + """ + Find Profiles in a Project + :param profile_schema: Profile Schema Name + :param attributes: Profile Schema Attributes + :param filter: filter + :param limit: limit + :return: List[ProfileSummary] + """ + variables = { + "project": self._project(), + "profileSchema": profile_schema, + "attributes": attributes, + } + if filter: + variables.update({"filter": filter}) + if limit: + variables.update({"limit": limit}) + profiles = self._query( + gql_queries.FIND_PROFILES, variables=variables, ip_type="profiles" + ) + return dicts_to_classes(profiles, Profile) + + def describe_profile( + self, profile_id: str, profile_schema: str + ) -> Optional[Profile]: + """ + Describe Profile + + :param profile_schema: Profile Schema Name + :param profile_id: Profile ID + :return: Profile + """ + schema = self._query( + gql_queries.PROFILE_BY_ID, + variables={ + "project": self._project(), + "profile": profile_id, + "schema": profile_schema, + }, + ip_type="profile", + ) + return dict_to_attr_class(schema, Profile) + + def describe_attribute_by_key( + self, profile_id: str, profile_schema: str, attribute_key: str + ) -> Optional[ProfileAttribute]: + """ + Describe a specific attribute in the profile ... + + :param profile_id: Profile ID + :param profile_schema: Profile Schema Name + :param attribute_key: Attribute Key + :return: ProfileAttribute + """ + profile = self.describe_profile(profile_id, profile_schema) + if not profile: + return None + return head([a for a in profile.attributes if a.key == attribute_key]) + + def get_profile_count(self, profile_schema: str, filter: str = None) -> int: + """ + Get Profiles Count for a Schema + + :param profile_schema: Profile Schema Name + :param filter: Query Filter + :return: Count + """ + variables = {"project": self._project(), "schema": profile_schema} + if filter: + variables.update({"filter": filter}) + return self._query( + gql_queries.PROFILE_COUNT, variables=variables, ip_type="profileCount" + ) + + def list_profile_features( + self, profile_schema: str + ) -> Optional[List[ProfileFeature]]: + """ + List Profile Features + + :param profile_schema: Profile Schema Name + :return: List[ProfileFeature] + """ + profile_features = self._query( + gql_queries.PROFILE_FEATURES, + variables={"project": self._project(), "profileSchema": profile_schema}, + ip_type="profileFeatures", + ) + return dicts_to_classes(profile_features, ProfileFeature) + + def profile_group_count( + self, + profile_schema: str, + group_by: List[str], + filter: str = None, + limit: int = None, + ) -> Optional[List[GroupCount]]: + """ + + :param profile_schema: Profile Schema Name + :param group_by: GroupBy Key + :param filter: Query Filter + :param limit: Limit + :return: Grouped Count for each Attribute Value + """ + variables = { + "project": self._project(), + "schema": profile_schema, + "groupBy": group_by, + } + if filter: + variables.update({"filter": filter}) + if limit: + variables.update({"limit": limit}) + + group_count = self._query( + gql_queries.PROFILE_GROUP_COUNT, + variables=variables, + ip_type="profileGroupCount", + ) + return dicts_to_classes(group_count, GroupCount) + + def find_profiles_history( + self, profile_schema: str, limit: int = None + ) -> Optional[List[ProfileCommit]]: + """ + Find Profiles History + + :param profile_schema: Profile Schema Name + :param limit + :return: List[ProfileCommit] + """ + variables = {"project": self._project(), "profileSchema": profile_schema} + if limit: + variables.update({"limit": limit}) + + group_count = self._query( + gql_queries.PROFILE_HISTORY, variables=variables, ip_type="profileHistory" + ) + return dicts_to_classes(group_count, ProfileCommit) + + def find_profiles_for_plan( + self, + simulation_id: str, + profile_schema: str, + plan_id: str, + filter: str = None, + limit: int = None, + ) -> Optional[List[Profile]]: + """ + Find Profiles for a Plan by SimulationId and PlanId + + :param simulation_id: Campaign SimulationId + :param profile_schema: Profile Schema Name + :param plan_id: PlanID + :param filter + :param limit + :return: List of Profiles + """ + variables = { + "project": self._project(), + "profileSchema": profile_schema, + "simulationId": simulation_id, + "planId": plan_id, + } + if filter: + variables.update({"filter": filter}) + if limit: + variables.update({"limit": limit}) + + profiles = self._query( + gql_queries.PROFILES_FOR_PLAN, variables=variables, ip_type="profilesForPlan" + ) + return dicts_to_classes(profiles, Profile) + + def save_profile_schema(self, schema: ProfileSchema) -> Optional[ProfileSchema]: + """ + Saves a new profile schema ... + + :param schema: + :return: Profile Schema as dict + """ + # As Save Profile Schema GQL doesn't support saving bucketAttributes + # and customAttributes, we have to save them separately + schema_dict = schema.to_dict() + bucket_attributes = schema_dict.pop("bucketAttributes") + custom_attributes = schema_dict.pop("customAttributes") + # Save Schema without Calculated Attributes + r = self._query( + gql_queries.CREATE_PROFILE_SCHEMA, + variables={"input": schema_dict}, + ip_type="createProfileSchema", + ) + + # Save Bucket Attributes + if bucket_attributes: + self.save_bucket_attributes(schema.name, bucket_attributes) + + # Save Custom Attributes + if custom_attributes: + self.save_custom_attributes(schema.name, custom_attributes) + + # Build Profiles + self.build_profiles_from_schema(schema.name) + return dict_to_attr_class(r, ProfileSchema) + + def save_profile_schema_from_json( + self, schema_json: Union[str, dict] + ) -> Optional[ProfileSchema]: + """ + Saves a profile schema accepting input as Dictionary + + :param schema_json: Dictionary Object or Filepath + :return: Profile Schema as dict + """ + # As Save Profile Schema GQL doesn't support saving bucketAttributes + # and customAttributes, we have to save them separately + if isinstance(schema_json, str): + # Read file if filepath is Received + try: + schema_json = json.load(open(schema_json, "rb")) + except Exception as e: + print("Error Reading Schema File with Message: {}".format(e)) + return None + bucket_attributes = schema_json.pop("bucketAttributes") + custom_attributes = schema_json.pop("customAttributes") + + # Save Schema without Calculated Attributes + r = self._query( + gql_queries.CREATE_PROFILE_SCHEMA, + variables={"input": schema_json}, + ip_type="createProfileSchema", + ) + + # Save Bucket Attributes + if bucket_attributes: + self.save_bucket_attributes(schema_json.get("name"), bucket_attributes) + + # Save Custom Attributes + if custom_attributes: + self.save_custom_attributes(schema_json.get("name"), custom_attributes) + + # Build Profiles + self.build_profiles_from_schema(schema_json.get("name")) + return dict_to_attr_class(r, ProfileSchema) + + def save_bucket_attributes( + self, + profile_schema: str, + bucket_attributes: List[Union[BucketAttributeSpec, dict]], + ): + """ + Save Multiple Bucket Attributes for a Profile Schema + :param profile_schema: + :param bucket_attributes: + """ + for attribute in bucket_attributes: + self.save_bucket_attribute(profile_schema, attribute) + + def save_custom_attributes( + self, + profile_schema: str, + custom_attributes: List[Union[CustomAttributeSpec, dict]], + ): + """ + Save Multiple Custom Attributes for a Profile Schema + + :param profile_schema: + :param custom_attributes: + """ + for attribute in custom_attributes: + self.save_custom_attribute(profile_schema, attribute) + + def save_custom_attribute( + self, profile_schema: str, custom_attribute: Union[CustomAttributeSpec, dict] + ) -> Optional[ProfileSchema]: + """ + Create custom attribute for a profile schema + + :param profile_schema: Profile Schema + :param custom_attribute: CustomAttributeSpec + :return: ProfileSchema + """ + r = self._query( + gql_queries.CREATE_CUSTOM_ATTRIBUTE, + variables={ + "input": { + "profileSchema": profile_schema, + "project": self._project(), + "attribute": custom_attribute.to_dict() + if isinstance(custom_attribute, CustomAttributeSpec) + else custom_attribute, + } + }, + ip_type="createCustomAttribute", + ) + return dict_to_attr_class(r, ProfileSchema) + + def save_bucket_attribute( + self, profile_schema: str, bucket_attribute: Union[BucketAttributeSpec, dict] + ) -> Optional[ProfileSchema]: + """ + Create bucket attribute for a profile schema + + :param profile_schema: Profile Schema + :param bucket_attribute: BucketAttributeSpec + :return: ProfileSchema + """ + r = self._query( + gql_queries.CREATE_BUCKET_ATTRIBUTE, + variables={ + "input": { + "profileSchema": profile_schema, + "project": self._project(), + "attribute": bucket_attribute.to_dict() + if isinstance(bucket_attribute, BucketAttributeSpec) + else bucket_attribute, + } + }, + ip_type="createBucketAttribute", + ) + return dict_to_attr_class(r, ProfileSchema) + + def build_profiles_from_schema(self, profile_schema: str): + """ + Build profiles according to the specified ProfileSchema + + :param profile_schema: Profile Schema + :return: JobInfo + """ + job_info = self._query( + gql_queries.BUILD_PROFILES_FROM_SCHEMA, + variables={"project": self._project(), "profileSchema": profile_schema}, + ip_type="buildProfile", + ) + return dict_to_attr_class(job_info, JobInfo) + + def update_profiles(self, profile_schema: str, profiles: List[str]): + """ + Update Profiles by Profile Schema + + :param profile_schema: Profile Schema Name + :param profiles: List of Profiles as JSON Strings + :return: JobInfo + + Example Usage: + update_profiles(profile_schema="schema-name", + profiles="{'profile_id':'1D9A521F-8722-AF89-5CD5-37BC3D11111','Gender':'F', 'Phone':'(111) 555-7777', + 'Email':'cogntidsfk@email.com','ZipCode':'707009','State':'Texas','Date':'01/01/2000','Age':'41'}") + + """ + job_info = self._query( + gql_queries.UPDATE_PROFILES, + variables={ + "project": self._project(), + "profileSchema": profile_schema, + "profiles": profiles, + }, + ip_type="updateProfiles", + ) + return dict_to_attr_class(job_info, JobInfo) + + def update_custom_attribute( + self, profile_schema: str, custom_attribute: CustomAttributeSpec + ) -> Optional[ProfileSchema]: + """ + Update custom attribute for a profile schema + + :param profile_schema: + :param custom_attribute: CustomAttributeSpec + :return: ProfileSchema + """ + r = self._query( + gql_queries.UPDATE_CUSTOM_ATTRIBUTE, + variables={ + "input": { + "profileSchema": profile_schema, + "project": self._project(), + "attribute": custom_attribute.to_dict() + if isinstance(custom_attribute, CustomAttributeSpec) + else custom_attribute, + } + }, + ip_type="updateCustomAttribute", + ) + return dict_to_attr_class(r, ProfileSchema) + + def update_bucket_attribute( + self, profile_schema: str, bucket_attribute: BucketAttributeSpec + ) -> Optional[ProfileSchema]: + """ + Update bucket attribute for a profile schema + + :param profile_schema: + :param bucket_attribute: BucketAttributeSpec + :return: ProfileSchema + """ + r = self._query( + gql_queries.UPDATE_BUCKET_ATTRIBUTE, + variables={ + "input": { + "profileSchema": profile_schema, + "project": self._project(), + "attribute": bucket_attribute.to_dict() + if isinstance(bucket_attribute, BucketAttributeSpec) + else bucket_attribute, + } + }, + ip_type="updateBucketAttribute", + ) + return dict_to_attr_class(r, ProfileSchema) + + def update_bucket_attributes( + self, + profile_schema: str, + bucket_attributes: List[Union[BucketAttributeSpec, dict]], + ): + """ + Update Multiple Bucket Attributes for a Profile Schema + :param profile_schema: + :param bucket_attributes: + """ + for attribute in bucket_attributes: + self.update_bucket_attribute(profile_schema, attribute) + + def update_custom_attributes( + self, + profile_schema: str, + custom_attributes: List[Union[CustomAttributeSpec, dict]], + ): + """ + Update Multiple Custom Attributes for a Profile Schema + + :param profile_schema: + :param custom_attributes: + """ + for attribute in custom_attributes: + self.update_custom_attribute(profile_schema, attribute) + + def update_profile_schema(self, schema: ProfileSchema) -> Optional[ProfileSchema]: + """ + Updates an existing profile schema ... + + :param schema: ProfileSchema Object + :return: Profile Schema as dictionary + """ + # As Save Profile Schema GQL doesn't support saving bucketAttributes + # and customAttributes, we have to save them separately + schema_dict = schema.to_dict() + bucket_attributes = schema_dict.pop("bucketAttributes") + custom_attributes = schema_dict.pop("customAttributes") + r = self._query( + gql_queries.UPDATE_PROFILE_SCHEMA, + variables={"input": schema_dict}, + ip_type="updateProfileSchema", + ) + # Update Bucket Attributes + if bucket_attributes: + self.update_bucket_attributes(schema.name, bucket_attributes) + # Update Custom Attributes + if custom_attributes: + self.update_custom_attributes(schema.name, custom_attributes) + # Build Profiles + self.build_profiles_from_schema(schema.name) + return dict_to_attr_class(r, ProfileSchema) + + def update_profile_schema_from_json( + self, schema_json: Union[str, dict] + ) -> Optional[dict]: + """ + Updates an existing profile schema accepting input as Dictionary or Filepath + + :param schema_json: Dictionary Object or Filepath + :return: Profile Schema as dict + """ + if isinstance(schema_json, str): + # Read file from the path + try: + schema_json = json.load(open(schema_json, "rb")) + except Exception as e: + print("Error Reading Schema File with Message: {}".format(e)) + return None + bucket_attributes = schema_json.pop("bucketAttributes") + custom_attributes = schema_json.pop("customAttributes") + r = self._query( + gql_queries.UPDATE_PROFILE_SCHEMA, + variables={"input": schema_json}, + ip_type="updateProfileSchema", + ) + # Update Bucket Attributes + if bucket_attributes: + self.update_bucket_attributes(r.get("name"), bucket_attributes) + # Update Custom Attributes + if custom_attributes: + self.update_custom_attributes(r.get("name"), custom_attributes) + # Build Profiles + self.build_profiles_from_schema(r.get("name")) + return dict_to_attr_class(r, ProfileSchema) + + def delete_custom_attribute( + self, attribute_name, profile_schema + ) -> Optional[ProfileSchema]: + """ + Delete Custom Attribute by Name and Schema + :param attribute_name: Custom Attribute name + :param profile_schema: Profile Schema + :return: ProfileSchema + """ + r = self._query( + gql_queries.DELETE_CUSTOM_ATTRIBUTE, + variables={ + "input": { + "attributeName": attribute_name, + "profileSchema": profile_schema, + "project": self._project(), + } + }, + ip_type="createBucketAttribute", + ) + return dict_to_attr_class(r, ProfileSchema) + + def delete_bucket_attribute( + self, attribute_name, profile_schema + ) -> Optional[ProfileSchema]: + """ + Delete Bucket Attribute by Name and Schema + :param attribute_name: Bucket Attribute Name + :param profile_schema: Profile Schema Name + :return: ProfileSchema + """ + r = self._query( + gql_queries.DELETE_BUCKET_ATTRIBUTE, + variables={ + "input": { + "attributeName": attribute_name, + "profileSchema": profile_schema, + "project": self._project(), + } + }, + ip_type="createBucketAttribute", + ) + return dict_to_attr_class(r, ProfileSchema) + + def delete_schema(self, profile_schema: str) -> bool: + """ + Delete Profile Schema by Name + + :param profile_schema: Profile Schema Name + :return: True/False + """ + return self._query( + gql_queries.DELETE_PROFILE_SCHEMA, + variables={"input": {"project": self._project(), "name": profile_schema}}, + ip_type="deleteProfileSchema", + ) + + def delete_profile(self, profile_id: str, profile_schema: str) -> Optional[JobInfo]: + """ + Delete Profile by profileId and profileSchema + + :param profile_id: Profile Schema Name + :param profile_schema: Profile Schema Name + :return: True/False + """ + return self.delete_profiles( + profile_schema=profile_schema, + filter='profile_id.eq("{}")'.format(profile_id), + ) + + def delete_profiles(self, profile_schema: str, filter: str) -> Optional[JobInfo]: + """ + Delete Profiles by Profiles Schema and filter + + :param profile_schema: Profile Schema Name + :param filter: filter + :return: JobInfo + + Usage: + delete_profiles(profile_schema="abc", filter="profile_id.eq(10000800)") + delete_profiles(profile_schema="abc", filter="profile_id.isIn([10000800, 10001334,10001689,10002757])") + + """ + variables = {"project": self._project(), "profileSchema": profile_schema} + if filter: + variables.update({"filter": filter}) + + job_info = self._query( + gql_queries.DELETE_PROFILES, variables=variables, ip_type="deleteProfiles" + ) + return dict_to_attr_class(job_info, JobInfo) + + def delete_all_profiles(self, profile_schema: str) -> Optional[JobInfo]: + """ + Delete All Profiles for a Profiles Schema + + :param profile_schema: Profile Schema Name + :return: JobInfo + """ + job_info = self._query( + gql_queries.DELETE_ALL_PROFILES, + variables={"project": self._project(), "profileSchema": profile_schema}, + ip_type="deleteAllProfiles", + ) + return dict_to_attr_class(job_info, JobInfo) diff --git a/requirements-dev.txt b/requirements-dev.txt index df52dc6..3a4eb94 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,4 +9,7 @@ pylint>=2.16.1,<3 pytest-html>=3.2.0,<4 pytest-cov>=4.0.0,<5.0.0 cuid -black>=22.6.0,<24 \ No newline at end of file +black>=22.6.0,<24 +arrow~=0.15.0 +pydash~=4.7.6 +pandas~=1.5.3 \ No newline at end of file From 2aab17760ecc3d59bd84498df0dadcf8830182ef Mon Sep 17 00:00:00 2001 From: dvasani-CS Date: Tue, 28 Mar 2023 15:43:48 +0530 Subject: [PATCH 2/2] updated year --- cortex_common/__init__.py | 2 +- cortex_common/utils/assertion_utils.py | 2 +- cortex_common/utils/class_utils.py | 2 +- cortex_common/utils/config_utils.py | 2 +- cortex_common/utils/dataframe_utils.py | 2 +- cortex_common/utils/equality_utils.py | 2 +- cortex_common/utils/etl_utils.py | 2 +- cortex_common/utils/generator_utils.py | 2 +- cortex_common/utils/id_utils.py | 2 +- cortex_common/utils/object_utils.py | 2 +- cortex_common/utils/string_utils.py | 2 +- cortex_common/utils/time_utils.py | 2 +- cortex_common/utils/type_utils.py | 2 +- cortex_common/validators/__init__.py | 2 +- cortex_common/validators/lists.py | 2 +- cortex_profiles/datamodel/__init__.py | 2 +- cortex_profiles/datamodel/constants.py | 2 +- cortex_profiles/datamodel/dataframes.py | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cortex_common/__init__.py b/cortex_common/__init__.py index 729fa82..f94ec9d 100644 --- a/cortex_common/__init__.py +++ b/cortex_common/__init__.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/assertion_utils.py b/cortex_common/utils/assertion_utils.py index 9e0be8c..06aa7e1 100644 --- a/cortex_common/utils/assertion_utils.py +++ b/cortex_common/utils/assertion_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/class_utils.py b/cortex_common/utils/class_utils.py index 5425c40..dfbe1b0 100644 --- a/cortex_common/utils/class_utils.py +++ b/cortex_common/utils/class_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/config_utils.py b/cortex_common/utils/config_utils.py index 67a6481..b418daa 100644 --- a/cortex_common/utils/config_utils.py +++ b/cortex_common/utils/config_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/dataframe_utils.py b/cortex_common/utils/dataframe_utils.py index 10698d1..a2174d4 100644 --- a/cortex_common/utils/dataframe_utils.py +++ b/cortex_common/utils/dataframe_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/equality_utils.py b/cortex_common/utils/equality_utils.py index ca845da..b5243bc 100644 --- a/cortex_common/utils/equality_utils.py +++ b/cortex_common/utils/equality_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/etl_utils.py b/cortex_common/utils/etl_utils.py index 729fa82..f94ec9d 100644 --- a/cortex_common/utils/etl_utils.py +++ b/cortex_common/utils/etl_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/generator_utils.py b/cortex_common/utils/generator_utils.py index 6c05993..5c2a7d6 100644 --- a/cortex_common/utils/generator_utils.py +++ b/cortex_common/utils/generator_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/id_utils.py b/cortex_common/utils/id_utils.py index db82664..c955a1c 100644 --- a/cortex_common/utils/id_utils.py +++ b/cortex_common/utils/id_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/object_utils.py b/cortex_common/utils/object_utils.py index ba153d4..623c9c2 100644 --- a/cortex_common/utils/object_utils.py +++ b/cortex_common/utils/object_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/string_utils.py b/cortex_common/utils/string_utils.py index 4f49990..9d01cf0 100644 --- a/cortex_common/utils/string_utils.py +++ b/cortex_common/utils/string_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/time_utils.py b/cortex_common/utils/time_utils.py index c2fea7b..a046435 100644 --- a/cortex_common/utils/time_utils.py +++ b/cortex_common/utils/time_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/utils/type_utils.py b/cortex_common/utils/type_utils.py index 04b3c5d..ba11a96 100644 --- a/cortex_common/utils/type_utils.py +++ b/cortex_common/utils/type_utils.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/validators/__init__.py b/cortex_common/validators/__init__.py index a5310a2..d57de3b 100644 --- a/cortex_common/validators/__init__.py +++ b/cortex_common/validators/__init__.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_common/validators/lists.py b/cortex_common/validators/lists.py index 7948707..4e92fbe 100644 --- a/cortex_common/validators/lists.py +++ b/cortex_common/validators/lists.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_profiles/datamodel/__init__.py b/cortex_profiles/datamodel/__init__.py index 729fa82..f94ec9d 100644 --- a/cortex_profiles/datamodel/__init__.py +++ b/cortex_profiles/datamodel/__init__.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_profiles/datamodel/constants.py b/cortex_profiles/datamodel/constants.py index 94210da..25a3ed5 100644 --- a/cortex_profiles/datamodel/constants.py +++ b/cortex_profiles/datamodel/constants.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/cortex_profiles/datamodel/dataframes.py b/cortex_profiles/datamodel/dataframes.py index 5aea9fc..548673f 100644 --- a/cortex_profiles/datamodel/dataframes.py +++ b/cortex_profiles/datamodel/dataframes.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Cognitive Scale, Inc. All Rights Reserved. +Copyright 2021 Cognitive Scale, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.