Skip to content

Commit

Permalink
always log record count when validating
Browse files Browse the repository at this point in the history
  • Loading branch information
smnorris committed Aug 15, 2024
1 parent 16203d2 commit dc5e21a
Showing 1 changed file with 12 additions and 17 deletions.
29 changes: 12 additions & 17 deletions download.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,14 +87,14 @@ def validate_file(source):
)

# is there data?
alias = source["alias"]
if len(df.index) == 0:
count = len(df.index)
if count == 0:
raise ValueError(
f"Validation error: {source['alias']} - no data returned, check source and query"
)

# presume layer is defined correctly if no errors are raised
LOG.info(f"Validation successful: {alias}")
LOG.info(f"Validation successful: {source['alias']} - record count: {str(count)}")


def validate_bcgw(source):
Expand Down Expand Up @@ -127,20 +127,15 @@ def validate_bcgw(source):
f"Validation error: {source['alias']} - column {column} is not present in {table}, modify config 'field_mapper'"
)

# does query return values?
if source["query"]:
count = bcdata.get_count(table, query=source["query"])
if count > 0:
LOG.info(
f"Validation successful: {source['alias']} - query returns {str(count)} values"
)
if bcdata.get_count(table, query=source["query"]) == 0:
raise ValueError(
f"Validation error: {source['alias']} - no data returned, check query against {table}"
)
# is there data?
count = bcdata.get_count(table, query=source["query"])
if count == 0:
raise ValueError(
f"Validation error: {source['alias']} - no data returned, check query against {table}"
)

# that is it for validation, presume layer is defined correctly if no errors are raised
LOG.info(f"Validation successful: {source['alias']}")
# presume source is defined correctly if no errors are raised
LOG.info(f"Validation successful: {source['alias']} - record count: {str(count)}")


def to_multipart(df):
Expand Down Expand Up @@ -174,7 +169,7 @@ def validate_sources(sources, validate_data=True, alias=None):
elif source["source_type"] == "FILE":
validate_file(source)

LOG.info("Validation successful - all layers appear valid")
LOG.info("Validation successful: all layers appear valid")

# return validated (and indexed/dated) sources as ordered dictionary
return sources
Expand Down

0 comments on commit dc5e21a

Please sign in to comment.