Skip to content

create_table doesn't recognise __temp__ database #156

Description

The following works:

import pydbtools as pydb
import pandas as pd

df = pd.DataFrame({"x": [1,2,3]})
pydb.dataframe_to_temp_table(df, "xs")
pydb.read_sql_query("select * from __temp__.xs")

Subsequently

pydb.create_table(sql="select * from __temp__.xs", table="xs", database="mike_test", location="s3://alpha-everyone/mike_test/xs")

gives the error

---------------------------------------------------------------------------
QueryFailed                               Traceback (most recent call last)
Cell In[4], [line 1](vscode-notebook-cell:?execution_count=4&line=1)
----> [1](vscode-notebook-cell:?execution_count=4&line=1) pydb.create_table(sql="select * from __temp__.xs", table="xs", database="mike_test", location="s3://alpha-everyone/mike_test/xs")

File ~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/pydbtools/_wrangler.py:334, in create_table(sql, database, table, location, partition_cols, boto3_session)
    315 def create_table(
    316     sql: str,
    317     database: str,
   (...)    321     boto3_session=None,
    322 ):
    323     """
    324     Create a table in a database from a SELECT statement
    325 
   (...)    332         boto3_session: optional boto3 session
    333     """
--> [334](https://vscode-remote+mratford-002dvscode-002etools-002eanalytical-002dplatform-002eservice-002ejustice-002egov-002euk-003a443.vscode-resource.vscode-cdn.net/home/analyticalplatform/bold_rr_pilot_families_llm_info_extraction/~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/pydbtools/_wrangler.py:334)     return ath.create_ctas_table(
    335         sql=sql,
    336         database=database,
    337         ctas_database=database,
    338         ctas_table=table,
    339         s3_output=s3_path_join(location, table + ".parquet"),
    340         partitioning_info=partition_cols,
    341         wait=True,
    342         boto3_session=boto3_session,
    343     )

File ~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/_config.py:[712](https://vscode-remote+mratford-002dvscode-002etools-002eanalytical-002dplatform-002eservice-002ejustice-002egov-002euk-003a443.vscode-resource.vscode-cdn.net/home/analyticalplatform/bold_rr_pilot_families_llm_info_extraction/~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/_config.py:712), in apply_configs.<locals>.wrapper(*args_raw, **kwargs)
    710         del args[name]
    711         args = {**args, **keywords}
--> 712 return function(**args)

File ~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/athena/_utils.py:873, in create_ctas_table(sql, database, ctas_table, ctas_database, s3_output, storage_format, write_compression, partitioning_info, bucketing_info, field_delimiter, schema_only, workgroup, data_source, encryption, kms_key, categories, wait, athena_query_wait_polling_delay, execution_params, params, paramstyle, boto3_session)
    868         if "Column type is unknown" in msg:
    869             raise exceptions.InvalidArgumentValue(
    870                 "Please, don't leave undefined columns types in your query. You can cast to ensure it. "
    871                 "(E.g. 'SELECT CAST(NULL AS INTEGER) AS MY_COL, ...')"
    872             )
--> [873](https://vscode-remote+mratford-002dvscode-002etools-002eanalytical-002dplatform-002eservice-002ejustice-002egov-002euk-003a443.vscode-resource.vscode-cdn.net/home/analyticalplatform/bold_rr_pilot_families_llm_info_extraction/~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/athena/_utils.py:873)         raise ex
    874 else:
    875     response["ctas_query_id"] = query_execution_id

File ~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/athena/_utils.py:851, in create_ctas_table(sql, database, ctas_table, ctas_database, s3_output, storage_format, write_compression, partitioning_info, bucketing_info, field_delimiter, schema_only, workgroup, data_source, encryption, kms_key, categories, wait, athena_query_wait_polling_delay, execution_params, params, paramstyle, boto3_session)
    849 if wait:
    850     try:
--> [851](https://vscode-remote+mratford-002dvscode-002etools-002eanalytical-002dplatform-002eservice-002ejustice-002egov-002euk-003a443.vscode-resource.vscode-cdn.net/home/analyticalplatform/bold_rr_pilot_families_llm_info_extraction/~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/athena/_utils.py:851)         response["ctas_query_metadata"] = _get_query_metadata(
    852             query_execution_id=query_execution_id,
    853             boto3_session=boto3_session,
    854             categories=categories,
    855             metadata_cache_manager=_cache_manager,
    856             athena_query_wait_polling_delay=athena_query_wait_polling_delay,
    857         )
    858     except exceptions.QueryFailed as ex:
    859         msg: str = str(ex)

File ~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/athena/_utils.py:237, in _get_query_metadata(query_execution_id, boto3_session, categories, query_execution_payload, metadata_cache_manager, athena_query_wait_polling_delay, execution_params, dtype_backend)
    233     _query_execution_payload = query_execution_payload
    234 else:
    235     _query_execution_payload = cast(
    236         "QueryExecutionTypeDef",
--> [237](https://vscode-remote+mratford-002dvscode-002etools-002eanalytical-002dplatform-002eservice-002ejustice-002egov-002euk-003a443.vscode-resource.vscode-cdn.net/home/analyticalplatform/bold_rr_pilot_families_llm_info_extraction/~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/athena/_utils.py:237)         _executions.wait_query(
    238             query_execution_id=query_execution_id,
    239             boto3_session=boto3_session,
    240             athena_query_wait_polling_delay=athena_query_wait_polling_delay,
    241         ),
    242     )
    243 cols_types: dict[str, str] = get_query_columns_types(
    244     query_execution_id=query_execution_id, boto3_session=boto3_session
    245 )
    246 _logger.debug("Casting query column types: %s", cols_types)

File ~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/_config.py:712, in apply_configs.<locals>.wrapper(*args_raw, **kwargs)
    710         del args[name]
    711         args = {**args, **keywords}
--> [712](https://vscode-remote+mratford-002dvscode-002etools-002eanalytical-002dplatform-002eservice-002ejustice-002egov-002euk-003a443.vscode-resource.vscode-cdn.net/home/analyticalplatform/bold_rr_pilot_families_llm_info_extraction/~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/_config.py:712) return function(**args)

File ~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/athena/_executions.py:235, in wait_query(query_execution_id, boto3_session, athena_query_wait_polling_delay)
    233 _logger.debug("Query state change reason: %s", response["Status"].get("StateChangeReason"))
    234 if state == "FAILED":
--> [235](https://vscode-remote+mratford-002dvscode-002etools-002eanalytical-002dplatform-002eservice-002ejustice-002egov-002euk-003a443.vscode-resource.vscode-cdn.net/home/analyticalplatform/bold_rr_pilot_families_llm_info_extraction/~/bold_rr_pilot_families_llm_info_extraction/.venv/lib/python3.11/site-packages/awswrangler/athena/_executions.py:235)     raise exceptions.QueryFailed(response["Status"].get("StateChangeReason"))
    236 if state == "CANCELLED":
    237     raise exceptions.QueryCancelled(response["Status"].get("StateChangeReason"))

QueryFailed: SCHEMA_NOT_FOUND: line 7:3: Schema '__temp__' does not exist. You may need to manually clean the data at location 's3://alpha-everyone/mike_test/xs.parquet/tables/994f0ee2-bb12-46bf-bad4-45e0d0e5f7f2' before retrying. Athena will not delete data in your account.

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions