diff --git a/.gitignore b/.gitignore index 4b68dda..fd68c17 100644 --- a/.gitignore +++ b/.gitignore @@ -198,4 +198,6 @@ cython_debug/ # terraform modules *terraform* -builds \ No newline at end of file +builds + +test.py \ No newline at end of file diff --git a/README.md b/README.md index 99501a7..76c769c 100644 --- a/README.md +++ b/README.md @@ -199,3 +199,22 @@ pull the example focus csv: `curl -LO https://raw.githubusercontent.com/FinOps-O install [poetry](https://python-poetry.org/docs/#installation) testing: `make test` + +## built-in focus conversions + +users may contribute a platform-specific subclass of the focus object to handle special cases in their billing exports. + +### mongodb atlas + +```python +from harness_ccm_external_data import MongoDBAtlas + +atlas = MongoDBAtlas( + "MongoDB Atlas", + "My Company Inc.", + "usage-summary-8765434567887656789-20250201.csv", + harness_account_id=getenv("HARNESS_ACCOUNT_ID"), + harness_platform_api_key=getenv("HARNESS_PLATFORM_API_KEY"), +) +atlas.upload() +``` diff --git a/poetry.lock b/poetry.lock index 25c5025..4240076 100644 --- a/poetry.lock +++ b/poetry.lock @@ -129,6 +129,62 @@ files = [ [package.extras] all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +[[package]] +name = "numpy" +version = "2.0.2" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version < \"3.11\"" +files = [ + {file = "numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66"}, + {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b"}, + {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd"}, + {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318"}, + {file = "numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8"}, + {file = "numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326"}, + {file = "numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97"}, + {file = "numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57"}, + {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a"}, + {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669"}, + {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951"}, + {file = "numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9"}, + {file = "numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15"}, + {file = "numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4"}, + {file = "numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c"}, + {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c"}, + {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692"}, + {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a"}, + {file = "numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c"}, + {file = "numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded"}, + {file = "numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5"}, + {file = "numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b"}, + {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729"}, + {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1"}, + {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd"}, + {file = "numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d"}, + {file = "numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d"}, + {file = "numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa"}, + {file = "numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c"}, + {file = "numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385"}, + {file = "numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78"}, +] + [[package]] name = "numpy" version = "2.3.1" @@ -136,6 +192,7 @@ description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.11" groups = ["main"] +markers = "python_version >= \"3.11\"" files = [ {file = "numpy-2.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6ea9e48336a402551f52cd8f593343699003d2353daa4b72ce8d34f66b722070"}, {file = "numpy-2.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5ccb7336eaf0e77c1635b232c141846493a588ec9ea777a7c24d7166bb8533ae"}, @@ -243,7 +300,11 @@ files = [ ] [package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\""} +numpy = [ + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, +] python-dateutil = ">=2.8.2" pytz = ">=2020.1" tzdata = ">=2022.7" @@ -366,5 +427,5 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.1" -python-versions = ">=3.13" -content-hash = "5756156004de7a4ccef45bc30e95a2e9e5e1c6fa4048b7e8a5d6f63a1a3f0bdc" +python-versions = ">=3.9" +content-hash = "0bd6a37207c1ecff546a10dd287ebbe9057d6eab242faf56cd7f00da927b6a25" diff --git a/pyproject.toml b/pyproject.toml index 2dfd30d..9dc0826 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,12 @@ [project] name = "harness-ccm-external-data" -version = "0.1.6" +version = "0.2.0-rc.1" description = "Tools to help manage external data for Harness CCM" authors = [ {name = "Riley Snyder",email = "riley.snyder@harness.io"} ] readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "pandas (>=2.3.0,<3.0.0)", "requests (>=2.32.4,<3.0.0)" diff --git a/src/harness_ccm_external_data/__init__.py b/src/harness_ccm_external_data/__init__.py index bcb1769..5f4951b 100644 --- a/src/harness_ccm_external_data/__init__.py +++ b/src/harness_ccm_external_data/__init__.py @@ -1 +1,2 @@ from .focus_data import Focus, HARNESS_FIELDS +from .mongodb_atlas import MongoDBAtlas diff --git a/src/harness_ccm_external_data/focus_data.py b/src/harness_ccm_external_data/focus_data.py index 1ff1ad7..058a56d 100644 --- a/src/harness_ccm_external_data/focus_data.py +++ b/src/harness_ccm_external_data/focus_data.py @@ -72,10 +72,12 @@ def __init__( ): self.provider = provider # cloud platform self.data_source = data_source # instance of this cloud platform + self.source = source self.provider_type = provider_type self.invoice_period = invoice_period self.provider_uuid = provider_uuid - self.source = source + self.separator = separator + self.skip_rows = skip_rows self.cost_multiplier = cost_multiplier self.converters = converters self.additional_columns = {} @@ -90,6 +92,7 @@ def __init__( self.harness_account_id = harness_account_id # Stub for generated content + self.billing_content: pd.DataFrame = None self.harness_focus_content: pd.DataFrame = None # sanitize mappings @@ -122,7 +125,7 @@ def __init__( ) del self.mapping[field] - baseline_converters = { + self.baseline_converters = { # make sure provider is set self.mapping["ProviderName"]: lambda x: self.provider if not x @@ -130,31 +133,35 @@ def __init__( } if cost_multiplier: # apply given cost multiplier - baseline_converters[self.mapping["EffectiveCost"]] = ( + self.baseline_converters[self.mapping["EffectiveCost"]] = ( lambda x: pd.to_numeric(x) * cost_multiplier ) + def load_and_convert_data(self): + """ + Load in the billing data and apply any specified modifications + """ self.billing_content = ( self.source if isinstance(self.source, pd.DataFrame) else pd.read_csv( self.source, - sep=separator, + sep=self.separator, engine="python", - skiprows=skip_rows, + skiprows=self.skip_rows, # any converters specified by the user will override built-in ones - converters={**baseline_converters, **converters}, + converters={**self.baseline_converters, **self.converters}, ) ) - for field, value in self.additional_columns.items(): - self.billing_content[field] = value - - def render(self) -> pd.DataFrame: + def convert_fields(self) -> pd.DataFrame: """ - Create the Harness-aligned FOCUS CSV + Convert the billing data to a format that is compatible with Harness """ + if self.billing_content is None: + self.load_and_convert_data() + self.harness_focus_content = pd.DataFrame() for focus_field, source_field in self.mapping.items(): if source_field in self.billing_content.columns: @@ -165,6 +172,9 @@ def render(self) -> pd.DataFrame: # Default value for missing columns self.harness_focus_content[focus_field] = source_field + for field, value in self.additional_columns.items(): + self.harness_focus_content[field] = value + return self.harness_focus_content def render_file(self, filename: str): @@ -172,10 +182,12 @@ def render_file(self, filename: str): Save the Harness-CSV to a file """ + if self.billing_content is None: + self.load_and_convert_data() if self.harness_focus_content is None: - self.render().to_csv(filename, index=False) - else: - self.harness_focus_content.to_csv(filename, index=False) + self.convert_fields() + + self.harness_focus_content.to_csv(filename, index=False) def _list_providers(self): """ @@ -383,7 +395,7 @@ def _get_invoice_period(self) -> str: start_date_str = self.harness_focus_content["BillingPeriodStart"].iloc[0] # Parse the dates - start_date = datetime.strptime(start_date_str, "%Y-%m-%d %H:%M:%S") + start_date = datetime.strptime(start_date_str, "%Y-%m-%dT%H:%M:%S") # Calculate the first day of the month for the start date period_start = start_date.replace(day=1) @@ -425,7 +437,7 @@ def _trigger_ingestion(self, provider_id: str, invoice_periods: list) -> bool: def upload( self, harness_platform_api_key: str = None, harness_account_id: str = None - ): + ) -> str | None: """ Upload the Harness-CSV data to Harness @@ -434,7 +446,7 @@ def upload( harness_account_id (str): Account ID for Harness Returns: - bool: True if all steps completed successfully, False otherwise + str | None: Object name if all steps completed successfully, None otherwise """ if harness_platform_api_key: @@ -445,7 +457,7 @@ def upload( # Ensure we have the rendered content if self.harness_focus_content is None: - self.render() + self.convert_fields() csv_content = self.harness_focus_content.to_csv(index=False) md5_hash = self._get_md5_hash(csv_content) @@ -453,19 +465,19 @@ def upload( # Ensure we have a provider if self.provider_uuid is None: if not self._create_provider(): - return False + return None # If no invoice_period is provided, calculate it from the data this_invoice_period = self._get_invoice_period() if not this_invoice_period: print("Failed to determine invoice period from data") - return False + return None # Check if file has already been uploaded for file in self.list_files(): if file["md5"] == md5_hash: print(f"File already uploaded: {md5_hash}") - return False + return None # Generate a unique object name timestamp = datetime.now().strftime("%Y%m%d%H%M%S") @@ -479,12 +491,12 @@ def upload( ) if not signed_url: print("Failed to get signed URL") - return False + return None # Step 2: Upload to GCS if not self._upload_to_gcs(signed_url, csv_content): print("Failed to upload to GCS") - return False + return None # Extract the GCS URL from the signed URL (remove query parameters) cloud_storage_path = signed_url.split("?")[0] @@ -499,14 +511,14 @@ def upload( cloud_storage_path, ): print("Failed to mark upload as complete") - return False + return None # Step 4: Trigger ingestion if not self._trigger_ingestion(self.provider_uuid, [this_invoice_period]): print("Failed to trigger ingestion") - return False + return None - return True + return object_name def create_dataset(data: list(list()) = None) -> pd.DataFrame: """ diff --git a/src/harness_ccm_external_data/mongodb_atlas.py b/src/harness_ccm_external_data/mongodb_atlas.py new file mode 100644 index 0000000..d5f6981 --- /dev/null +++ b/src/harness_ccm_external_data/mongodb_atlas.py @@ -0,0 +1,131 @@ +from typing import Dict, Sequence, Optional +import hashlib +from datetime import datetime +from dateutil.relativedelta import relativedelta + +import pandas as pd + +from .focus_data import Focus + +MONGODB_ATLAS_MAPPING = { + "BillingAccountId": "Organization ID", + "BillingAccountName": "Organization Name", + "BillingPeriodEnd": "Date", + "BillingPeriodStart": "Date", + "ChargeCategory": "Note", + "ChargePeriodStart": "Usage Date", + "ChargePeriodEnd": "Date", + "ConsumedQuantity": "Quantity", + "EffectiveCost": "Amount", + "ResourceId": "Cluster", + "RegionName": "Region", + "ServiceName": "SKU", + "SubAccountId": "Project ID", + "SkuId": "SKU", + "SubAccountName": "Project Name", +} + + +class MongoDBAtlas(Focus): + def __init__( + self, + provider: str, + data_source: str, + source: str | pd.DataFrame, + provider_type: str = "CUSTOM", + invoice_period: str = "MONTHLY", + provider_uuid: str = None, + mapping: Dict[str, str] = {}, + separator: str = ",", + skip_rows: int | Sequence[int] = 5, + cost_multiplier: float = 1.0, + converters: Dict[str, callable] = {}, + additional_columns: Dict[str, str] = {}, + validate: bool = True, + harness_platform_api_key: str = None, + harness_account_id: str = None, + # mongodb atlas specific + ## tag column start is the column number of the first tag column + tag_column_start: int = 19, + ## resource id column start is the column number of the cluster column, used to create a resource id + resource_id_column_start: int = 10, + ## resource id column end is the column number of the application column, used to create a resource id + resource_id_column_end: int = 14, + ): + super().__init__( + provider, + data_source, + source, + provider_type, + invoice_period, + provider_uuid, + MONGODB_ATLAS_MAPPING | mapping, + separator, + skip_rows, + cost_multiplier, + {MONGODB_ATLAS_MAPPING["ChargeCategory"]: lambda x: "Credit" if "credit" in x.lower() else "Usage"} | converters, + additional_columns, + validate, + harness_platform_api_key, + harness_account_id, + ) + self.tag_column_start = tag_column_start + self.resource_id_column_start = resource_id_column_start + self.resource_id_column_end = resource_id_column_end + + def convert_fields(self) -> pd.DataFrame: + super().convert_fields() + + # create resource id using Cluster/Replica Set/Config Server/Application + self.harness_focus_content["ResourceId"] = self.billing_content[ + self.billing_content.columns[ + self.resource_id_column_start:self.resource_id_column_end + ] + ].apply(lambda x: "/".join(x.dropna().astype(str)), axis=1) + + # Take individual tag columns and combine them into the focus_data format dictionary. + self.harness_focus_content["Tags"] = self.billing_content[ + self.billing_content.columns[self.tag_column_start:] + ].apply( + lambda x: { + key.split("/")[1]: str(value) + for key, value in x.to_dict().items() + if not pd.isna(value) + }, + axis=1, + ) + + self.harness_focus_content["BillingPeriodStart"] = self.harness_focus_content[ + "BillingPeriodStart" + ].apply( + lambda x: datetime.strptime(x, "%m/%d/%Y") + .replace(day=1) + .strftime("%Y-%m-%dT00:00:00") + ) + + self.harness_focus_content["BillingPeriodEnd"] = self.harness_focus_content[ + "BillingPeriodEnd" + ].apply( + lambda x: ( + datetime.strptime(x, "%m/%d/%Y").replace(day=1) + + relativedelta(months=1) + ).strftime("%Y-%m-%dT00:00:00") + ) + + self.harness_focus_content["ChargePeriodStart"] = self.harness_focus_content[ + "ChargePeriodStart" + ].apply( + lambda x: datetime.strptime(x, "%m/%d/%Y") + .replace(day=1) + .strftime("%Y-%m-%dT%H:%M:%S") + ) + + self.harness_focus_content["ChargePeriodEnd"] = self.harness_focus_content[ + "ChargePeriodEnd" + ].apply( + lambda x: ( + datetime.strptime(x, "%m/%d/%Y") + relativedelta(days=1) + ).strftime("%Y-%m-%dT%H:%M:%S") + ) + + return self.harness_focus_content diff --git a/tests/test_focus_data_creation.py b/tests/test_focus_data_creation.py index d721e6b..44a906e 100644 --- a/tests/test_focus_data_creation.py +++ b/tests/test_focus_data_creation.py @@ -53,9 +53,11 @@ def test_passing_df(): df = Focus.create_dataset(TEST_DATA) test = Focus("MyTestPlatform", "Test", df) + test.load_and_convert_data() + assert set(HARNESS_FIELDS).issubset(set(test.billing_content.columns)) - test.render() + test.convert_fields() assert test.harness_focus_content is not None assert not test.harness_focus_content.empty diff --git a/tests/test_focus_data_load_transform.py b/tests/test_focus_data_load_transform.py index fff5d77..5c82020 100644 --- a/tests/test_focus_data_load_transform.py +++ b/tests/test_focus_data_load_transform.py @@ -9,7 +9,7 @@ test_data_length = 1000 test_data_col = 44 -test_data.render() +test_data.convert_fields() test_data.render_file(f"harness_{SAMPLE_DATA}") @@ -47,6 +47,7 @@ def test_data_load_skip_lines(): "MyTestPlatform", "Test", SAMPLE_DATA, skip_rows=to_skip_lines ) + test_data_skip_lines.convert_fields() raw_row, raw_col = test_data_skip_lines.billing_content.shape assert raw_row == test_data_length - to_skip_lines assert raw_col == test_data_col @@ -58,6 +59,7 @@ def test_data_load_skip_specific_lines(): "MyTestPlatform", "Test", SAMPLE_DATA, skip_rows=to_skip_specific ) + test_data_skip_specific.convert_fields() raw_row, raw_col = test_data_skip_specific.billing_content.shape assert raw_row == test_data_length - len(to_skip_specific) assert raw_col == test_data_col @@ -71,7 +73,7 @@ def test_data_load_multiply(): ) baseline_sum = pd.read_csv(SAMPLE_DATA)["EffectiveCost"].sum() - test_data_skip.render() + test_data_skip.convert_fields() assert ( test_data_skip.harness_focus_content["EffectiveCost"].sum() @@ -88,6 +90,8 @@ def test_data_convert(): converters={"BillingAccountId": lambda x: "FakeAccountID"}, ) + test_data_convert.load_and_convert_data() + assert ( test_data_convert.billing_content.iloc[0]["BillingAccountId"] == "FakeAccountID" ) @@ -116,6 +120,8 @@ def test_data_null_provider(tmpdir): filename, ) + test_data_provider.load_and_convert_data() + assert ( test_data_provider.billing_content.iloc[0]["ProviderName"] == "MyTestPlatformTest" @@ -130,7 +136,7 @@ def test_data_mapping(tmpdir): test_data_remap = Focus("MyTestPlatform", "Test", filename, mapping=mapped_col) - test_data_remap.render() + test_data_remap.convert_fields() assert len(test_data_remap.harness_focus_content["SkuId"]) == test_data_length @@ -143,7 +149,7 @@ def test_data_mapping_empty(tmpdir): mapping={}, ) - assert test_data_empty.render() is not None + assert test_data_empty.convert_fields() is not None test_data_none = Focus( "MyTestPlatform", @@ -152,7 +158,7 @@ def test_data_mapping_empty(tmpdir): mapping=None, ) - assert test_data_none.render() is not None + assert test_data_none.convert_fields() is not None def test_additional_columns(tmpdir): @@ -163,7 +169,7 @@ def test_additional_columns(tmpdir): additional_columns={"ConsumedQuantity": 1, "NonFocusField": "Value"}, ) - assert test_data_add.render() is not None + assert test_data_add.convert_fields() is not None assert test_data_add.harness_focus_content.iloc[0]["ConsumedQuantity"] == 1 assert ( test_data_add.harness_focus_content.iloc[test_data_length - 1][ diff --git a/tests/test_focus_data_upload.py b/tests/test_focus_data_upload.py index 15efecc..0824446 100644 --- a/tests/test_focus_data_upload.py +++ b/tests/test_focus_data_upload.py @@ -33,8 +33,8 @@ def test_data_upload(): SAMPLE_DATA, cost_multiplier=random.uniform(1.0, 2.0), converters={ - "BillingPeriodStart": lambda _: "2025-5-01 00:00:00", - "BillingPeriodEnd": lambda _: "2025-6-01 00:00:00", + "BillingPeriodStart": lambda _: "2025-5-01T00:00:00", + "BillingPeriodEnd": lambda _: "2025-6-01T00:00:00", "BillingAccountId": lambda _: billing_account_id, "BillingAccountName": lambda _: "Billing Account Name", "SubAccountId": lambda _: random.randint(100000000000, 999999999999), @@ -44,7 +44,7 @@ def test_data_upload(): harness_platform_api_key=getenv("HARNESS_PLATFORM_API_KEY"), ) - assert may_data.upload() is True + assert may_data.upload() is not None june_data = Focus( "Testing Upload", @@ -52,8 +52,8 @@ def test_data_upload(): SAMPLE_DATA, cost_multiplier=random.uniform(1.0, 2.0), converters={ - "BillingPeriodStart": lambda _: "2025-6-01 00:00:00", - "BillingPeriodEnd": lambda _: "2025-7-01 00:00:00", + "BillingPeriodStart": lambda _: "2025-6-01T00:00:00", + "BillingPeriodEnd": lambda _: "2025-7-01T00:00:00", "BillingAccountId": lambda _: billing_account_id, "BillingAccountName": lambda _: "Billing Account Name", "SubAccountId": lambda _: random.randint(100000000000, 999999999999), @@ -64,4 +64,4 @@ def test_data_upload(): harness_platform_api_key=getenv("HARNESS_PLATFORM_API_KEY"), ) - assert june_data.upload() is True + assert june_data.upload() is not None