December Development

.github/workflows/adf-cicd.yml

@@ -21,7 +21,7 @@ jobs:
     - name: Setup Node.js environment
       uses: actions/setup-node@v3.4.1
       with:
-        node-version: 18.x
+        node-version: 22.x
 
     - name: install ADF Utilities package
       run: npm install

.gitignore

@@ -260,6 +260,12 @@ paket-files/
 # PyTest coverage report
 .coverage
 
+# Ignore Python cache files
+**/*/__pycache__/*
+**/*/__pycache__/
+**/*.py[cod]
+**/*.pyc
+
 # Function App Local Publish files
 publishFunctions
 funcapp.zip

infrastructure/deployment/check_params_from_file.ps1

@@ -0,0 +1,23 @@
+#Assigining the parameters for the environment
+param(
+    [Parameter(Mandatory=$false)]
+    [string] $parametersFile
+)
+
+# Read all lines, ignore comments and 'using' statements
+$lines = Get-Content $parametersFile | Where-Object {
+    $_ -notmatch '^\s*//' -and $_ -notmatch '^\s*using'
+}
+
+# Extract param name and value
+$params = foreach ($line in $lines) {
+    if ($line -match 'param\s+(\w+)\s*=\s*(.+)') {
+        [PSCustomObject]@{
+            Name  = $matches[1]
+            Value = $matches[2].Trim()
+        }
+    }
+}
+
+# Render as table
+$params | Format-Table -AutoSize

infrastructure/deployment/deploy_azure_functions.ps1

@@ -7,7 +7,10 @@ param(
     [string] $resourceGroupName,
 
     [Parameter(Mandatory=$true)]
-    [string] $functionAppName
+    [string] $functionAppName,
+
+    [Parameter(Mandatory=$true)]
+    [string] $keyVaultName
 )
 
 # This command cleans the build output of the specified project using the Release configuration.

infrastructure/deployment/deploy_data_factory_components.ps1

@@ -3,6 +3,9 @@ param(
     [Parameter(Mandatory=$true)]
     [string] $tenantId,
 
+    [Parameter(Mandatory=$true)]
+    [string] $subscriptionId,
+
     [Parameter(Mandatory=$true)]
     [string] $location,
 
@@ -12,8 +15,11 @@ param(
     [Parameter(Mandatory=$true)]
     [string] $dataFactoryName
 )
+
 # Modules
 # Install-Module -Name "Az"
+# az login --tenant $tenantId
+
 # Import-Module -Name "Az"
 
 # Install-Module -Name "Az.DataFactory"
@@ -25,13 +31,16 @@ Import-Module -Name azure.datafactory.tools
 
 # Get Deployment Objects and Params files
 $scriptPath = (Join-Path -Path (Get-Location) -ChildPath "src/azure.datafactory") 
+
 $scriptPath = (Get-Location).Path -replace 'infrastructure\\deployment',''
+
 $scriptPath += "\src\azure.datafactory"
 
+
 $options = New-AdfPublishOption
 $options.CreateNewInstance = $false # New ADF workspace deployment not required.
 $options.Excludes.Add("trigger.*","")
 $options.Excludes.Add("factory.*","")
 
-
-Publish-AdfV2FromJson -RootFolder "$scriptPath" -ResourceGroupName "$resourceGroupName" -DataFactoryName "$dataFactoryName" -Location "$location" -Option $options -Stage "install"
+Set-AzContext -Subscription $subscriptionId
+Publish-AdfV2FromJson -RootFolder "$scriptPath" -ResourceGroupName "$resourceGroupName" -DataFactoryName "$dataFactoryName" -Location "$location" -Option $options -Stage "install"

infrastructure/deployment/deploy_wrapper.ps1

@@ -19,6 +19,22 @@ param(
 # Login to the Azure Tenant
 # az login --tenant $tenantId
 
+$currentLocation = Split-Path -Path $MyInvocation.MyCommand.Path -Parent
+$checkParamsScript = $currentLocation + '\check_params_from_file.ps1'
+& $checkParamsScript `
+    -parametersFile $parametersFile
+
+$acceptInput = Read-Host "Are the utilised parameters correct? (Y) yes, (all other input) no. Press enter to confirm."
+
+if ( $acceptInput.ToUpper() -eq "Y")
+{
+    Write-Host "Proceed with deployment"
+}
+else {
+    Write-Host "Cancelling deployment"
+    exit
+}
+
 # DEMO: Start a timer
 $processTimerStart = [System.Diagnostics.Stopwatch]::StartNew()
 
@@ -28,7 +44,6 @@ $bicepDeployment = az deployment sub create `
     --location $location `
     --template-file $templateFile `
     --parameters $parametersFile `
-    # --what-if
     | ConvertFrom-Json
 
 # Save Outputs of reusable details from BiCep for other scripts
@@ -48,8 +63,7 @@ $sqlDatabaseName = $bicepDeployment.properties.outputs.sqlDatabaseName.value
 $currentLocation = Split-Path -Path $MyInvocation.MyCommand.Path -Parent
 
 # Get Subscription Id from Name
-$subscriptionDetails = az account subscription list | ConvertFrom-Json 
-$subscriptionIdValue = $subscriptionDetails.subscriptionId
+$subscriptionIdValue = az account list --query "[?name=='${subscriptionId}'].id" --output tsv
 
 # Grant User Key Vault Secret Administrator RBAC to save Function App Key to KV
 $userDetails = az ad signed-in-user show | ConvertFrom-Json
@@ -67,7 +81,8 @@ $deployAzureFunctionsScript = $currentLocation + '\deploy_azure_functions.ps1'
 & $deployAzureFunctionsScript `
     -currentLocation $currentLocation `
     -resourceGroupName $resourceGroupName `
-    -functionAppName $functionAppName
+    -functionAppName $functionAppName `
+    -keyVaultName $keyVaultName
 
 
 # Set environment variables for Data Factory LS deployments:
@@ -82,6 +97,7 @@ $Env:KEYVAULT = $keyVaultName
 $deployDataFactoryComponentsScript = $currentLocation + '\deploy_data_factory_components.ps1'
 & $deployDataFactoryComponentsScript `
     -tenantId $tenantId `
+    -subscriptionId $subscriptionId `
     -location $location `
     -resourceGroupName $resourceGroupName `
     -dataFactoryName $dataFactoryName

src/azure.datafactory/dataset/Ingest_DS_Amazon_S3_Parquet.json

@@ -0,0 +1,67 @@
+{
+	"name": "Ingest_DS_Amazon_S3_Parquet",
+	"properties": {
+		"linkedServiceName": {
+			"referenceName": "Ingest_LS_AmazonS3_AKAuth",
+			"type": "LinkedServiceReference",
+			"parameters": {
+				"ServiceURL": {
+					"value": "@dataset().DSServiceURL",
+					"type": "Expression"
+				},
+				"SecretKeyName": {
+					"value": "@dataset().DSSecretKeyName",
+					"type": "Expression"
+				},
+				"AccessKeyId": {
+					"value": "@dataset().DSAccessKeyId",
+					"type": "Expression"
+				}
+			}
+		},
+		"parameters": {
+			"DSAccessKeyId": {
+				"type": "string"
+			},
+			"DSSecretKeyName": {
+				"type": "string"
+			},
+			"DSServiceURL": {
+				"type": "string"
+			},
+			"DSBucket": {
+				"type": "string"
+			},
+			"DSDirectory": {
+				"type": "string"
+			},
+			"DSFileName": {
+				"type": "string"
+			}
+		},
+		"folder": {
+			"name": "Ingest"
+		},
+		"annotations": [],
+		"type": "Parquet",
+		"typeProperties": {
+			"location": {
+				"type": "AmazonS3Location",
+				"bucketName": {
+					"value": "@dataset().DSBucket",
+					"type": "Expression"
+				},
+				"folderPath": {
+					"value": "@dataset().DSDirectory",
+					"type": "Expression"
+				},
+				"fileName": {
+					"value": "@dataset().DSFileName",
+					"type": "Expression"
+				}
+			},
+			"compressionCodec": "snappy"
+		},
+		"schema": []
+	}
+}

src/azure.datafactory/linkedService/Common_LS_cumulusdatabase.json

@@ -9,7 +9,7 @@
 		],
 		"type": "AzureSqlDatabase",
 		"typeProperties": {
-			"connectionString": "Integrated Security=False;Encrypt=True;Connection Timeout=30;Data Source=cfcdemodevsqldbuks01.database.windows.net;Initial Catalog=metadata-db"
+			"connectionString": "Integrated Security=False;Encrypt=True;Connection Timeout=30;Data Source=cfcumulusdevsqluks25.database.windows.net;Initial Catalog=cfcumulusdevsqldbuks25"
 		}
 	}
 }

src/azure.datafactory/linkedService/Ingest_LS_AmazonS3_AKAuth.json

@@ -0,0 +1,34 @@
+{
+	"name": "Ingest_LS_AmazonS3_AKAuth",
+	"properties": {
+		"parameters": {
+			"ServiceURL": {
+				"type": "string"
+			},
+			"SecretKeyName": {
+				"type": "string"
+			},
+			"AccessKeyId": {
+				"type": "string"
+			}
+		},
+		"annotations": [],
+		"type": "AmazonS3",
+		"typeProperties": {
+			"serviceUrl": "@{linkedService().ServiceURL}",
+			"accessKeyId": "@{linkedService().AccessKeyId}",
+			"secretAccessKey": {
+				"type": "AzureKeyVaultSecret",
+				"store": {
+					"referenceName": "Common_LS_cumuluskeys",
+					"type": "LinkedServiceReference"
+				},
+				"secretName": {
+					"value": "@linkedService().SecretKeyName",
+					"type": "Expression"
+				}
+			},
+			"authenticationType": "AccessKey"
+		}
+	}
+}