Setting up an Azure Data Lake and Azure Data Factory using Powershell

Login-AzureRmAccount
#first ensure that you have an Azure Data Lake that you want to use for ODX
#$resourceGroups = Get-AzureRmResourceGroup
#$azureDataLakeNames = "";
# foreach ($resourceGroup in $resourceGroups) {
# $azureDataLake = Get-AzureRmDataLakeStoreAccount -ResourceGroupName $resourceGroup.ResourceGroupName
#$azureDataLake
# $azureDataLakeName = $azureDataLake.Name
# $azureDataLakeNameLength = $azureDataLakeName.Length
# $azureDataLakeNameLength -gt 0
# if ($azureDataLakeNameLength -gt 0) {
# $azureDataLakeNames += " " + $azureDataLake.Name + " (resource group: " + $resourceGroup.ResourceGroupName + " & location: " + $resourceGroup.Location + ")"
# }
# }
# "-----------"
#"DataLakeNames: " + $azureDataLakeNames
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
#REQUIRED: you must enter a unique appname which will be used as the security principal
$appname = "inventose"
#OPTIONAL: change the password for the security principal password
$password = "Xyzpdq"
#run the above script, and replace DATALAKESTORENAME with the appropriate name/rg/location from your existing data lake store; or enter a new name to have a data lake created
$dataLakeStoreName = $appname
$odxResourceGroup = $appname
$dataLakeLocation = "Central US" #Central US, East US 2, North Europe
#recommended to use the same resource group as the data factory for simplicity, but you can use any resource group or enter a new name to create
$dataFactoryResourceGroup = $dataLakeStoreResourceGroup
#specify where you want your data factory - current options are East US, North Europe, West Central US, and West US
$dataFactoryLocation = "West US"
#-------------------------------------------------------------------------------
#-------------------------------------------------------------------------------
#create odxResourceGroup, if it does not exist
Get-AzureRmResourceGroup -Name $odxResourceGroup -ErrorVariable notPresent1 -ErrorAction 0
if ($notPresent1)
{
New-AzureRmResourceGroup -Location $dataLakeLocation -Name $odxResourceGroup
}
#create data lake, if it does not exist
Get-AzureRmDataLakeStoreAccount -Name $dataLakeStoreName -ErrorVariable notPresent2 -ErrorAction 0
if ($notPresent2)
{
New-AzureRmDataLakeStoreAccount -Location $dataLakeLocation -Name $dataLakeStoreName -ResourceGroupName $odxResourceGroup
}
$homepage = "https://ODXPS.com/" + $appname
#create security principal, if it does not exist
$app = New-AzureRmADApplication -DisplayName $appname -HomePage $homepage -IdentifierUris $homepage -Password $password
$app = Get-AzureRmADApplication -DisplayName $appname
$servicePrincipal = New-AzureRmADServicePrincipal -ApplicationId $app.ApplicationId
Start-Sleep 10
New-AzureRmRoleAssignment -RoleDefinitionName "Contributor" -Id $servicePrincipal.Id -ResourceGroupName $odxResourceGroup
New-AzureRmRoleAssignment -RoleDefinitionName "Data Factory Contributor" -Id $servicePrincipal.Id -ResourceGroupName $odxResourceGroup
New-AzureRmRoleAssignment -RoleDefinitionName "Reader" -Id $servicePrincipal.Id -ResourceGroupName $odxResourceGroup
#Set-AzureRmDataLakeStoreItemAclEntry -AccountName $dataLakeStoreName -Path / -AceType User -Id $app.ApplicationId -Permissions All
Set-AzureRmDataLakeStoreItemAclEntry -AccountName $dataLakeStoreName -Path / -AceType User -Id $servicePrincipal.Id -Permissions All
Get-AzureRmDataLakeStoreItem -Account $dataLakeStoreName -Path /ODX -ErrorVariable notPresent3 -ErrorAction 0
if ($notPresent3)
{
New-AzureRmDataLakeStoreItem -Folder -AccountName $dataLakeStoreName -Path /ODX
}
Set-AzureRmDataLakeStoreItemAclEntry -AccountName $dataLakeStoreName -Path /ODX -AceType User -Id $servicePrincipal.Id -Permissions All
#Start-Sleep 60 #there seems to be a lag between when these permissions are added and when they are applied...trying 1 minutes to start
$subscription = Get-AzureRmSubscription
$subscriptionId= ($subscription).Id
$tenantId = ($subscription).TenantId
#ensure there are permissions
#Get-AzureRmDataLakeStoreItemAclEntry -Account $dataLakeStoreName -Path /
#get information on datalake
$dataLake = Get-AzureRmDataLakeStoreAccount -Name $dataLakeStoreName
#here is a printout
"---------------------------------------------------------------"
"---------------------------------------------------------------"
$text1= "Azure Data Lake Name: " + $dataLakeStoreName + "`r`n" +
"Tenant ID: " + $tenantId + "`r`n" +
"Client ID: " + $app.ApplicationId + "`r`n" +
"Client Secret: " + $password + "`r`n" +
"Subscription ID: " + $subscriptionId + "`r`n" +
"Resource Group Name: " + $odxResourceGroup + "`r`n" +
"Data Lake URL: adl://" + $dataLake.Endpoint + "`r`n" +
"Location: " + $dataFactoryLocation
"---------------------------------------------------------------"
"---------------------------------------------------------------"
Out-File C:UsersMattDyorDesktopDataLake.ps1 -InputObject $text1

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s