locked
How to increase performance when running script against 100's of mailboxes RRS feed

  • Question

  • Hi there - hope this is the right forum to post - was split between chossing the powershell and tools forum and this one - went with this one.  So I have an EWS script which looks at mailboxes across onpremise and o365 and returns information about a users folders and retention tags applied as well as item count. This all works - but the request I have is to run this again possible a 1000 mailboxes.  So I am trying to figure out how to do this in a timely manner - i have come across the terms - synchronous and asynchronous - with various explanations - but basically my understanding is at the moment my script is running synchronously or basically processing 1 mailbox at a time from a collection of mailboxes..  asynchronously i presume is able to process multiple mailboxes at the same time - thereby increasing the performance and time to complete?  Don't know if this is a very simple and/or wrong view....  The code for my script is below - just looking for help on how to get a big performance increase cos at the moment for 1000 mailboxes or so it could take hours:






    [CmdletBinding()]

        param(

        [Parameter(
        Position = 0,
        Mandatory=$true,
        ValueFromPipelineByPropertyName=$true
        
        )]
        [String[]]
        $PrimarySmtpAddress

        

    )



    begin{
    <#
        if (-not (Get-PSSnapin | where {$_.Name -eq "Microsoft.Exchange.Management.PowerShell.SnapIn"} )) {
            Write-Host "Adding Exchange 2013 PowerShell role"
            Add-PSSnapin Microsoft.Exchange.Management.PowerShell.SnapIn

            }

      #>    
            
            $startTime = Get-Date
            $error.clear()
            [string]$LogFile = "C:\Temp\Log.txt"  
            
            $o365credential = get-credential -credential 'o365creds'
            
            $ExchangeSession = New-PSSession -ConfigurationName Microsoft.Exchange -ConnectionUri "https://outlook.office365.com/powershell-liveid/" -Credential $o365credential -Authentication "Basic" -AllowRedirection
            Import-PSSession $ExchangeSession -Prefix o365 -AllowClobber

            $credential = Get-Credential "onpremcreds"


            Add-Type -Path "C:\Program Files\Microsoft\Exchange\Web Services\2.0\Microsoft.Exchange.WebServices.dll"
            $exchangeService = New-Object Microsoft.Exchange.WebServices.Data.ExchangeService

            $Global:retentioninfo = @()
            $Global:RetentionTags = @()
            if($LogFile)
            {
                Remove-Item $LogFile -ea SilentlyContinue
            }
            
    }



    process {
       
      
      

      foreach($user in $PrimarySmtpAddress){
           
                 
                   function folderretention()
                   {
                            $FPageSize = 100
                            $FOffset = 0
                            $MoreItems =$true
                            $ItemCount=0
                            $folderView = new-object Microsoft.Exchange.WebServices.Data.FolderView($FPageSize,$FOffset,[Microsoft.Exchange.WebServices.Data.OffsetBasePoint]::Beginning)
                            $folderView.Traversal = [Microsoft.Exchange.WebServices.Data.FolderTraversal]::Deep
                            $oFindFolders = $exchangeService.FindFolders([Microsoft.Exchange.WebServices.Data.WellKnownFolderName]::MsgFolderRoot,$null,$folderView)
                            #?{(($_.DisplayName -notlike 'Calendar') -and ($_.DisplayName -notlike '*contacts') -and ($_.DisplayName -notlike '*recipient*'))}
                
                            $Global:RetentionTags = $exchangeService.GetUserRetentionPolicyTags()
                    

                                function GetTagName($tagGUID) {
                                if (!$tagGUID) { return ($RetentionTags.RetentionPolicyTags | ? {$_.Type -eq "All"}).DisplayName }
                                foreach ($tag in $RetentionTags.RetentionPolicyTags) {
                                if ($tag.RetentionId -eq $tagGUID ) { return $tag.DisplayName }
                                }
                            }


                                function GetRetentionAction($tagGUID)  {
                                if (!$tagGUID) { return ($RetentionTags.RetentionPolicyTags | ? {$_.Type -eq "All"}).RetentionAction }
                                foreach ($tag in $RetentionTags.RetentionPolicyTags) {
                                if ($tag.RetentionId -eq $tagGUID ) { return $tag.RetentionAction }

                                }
                            }

                                 function GetRetentionPeriod($tagGUID)  {
                                if (!$tagGUID) { return ($RetentionTags.RetentionPolicyTags | ? {$_.Type -eq "All"}).RetentionPeriod }
                                foreach ($tag in $RetentionTags.RetentionPolicyTags) {
                                if ($tag.RetentionId -eq $tagGUID ) { return $tag.RetentionPeriod }

                                }
                            }
                                
                           

                                $itemView = new-object Microsoft.Exchange.WebServices.Data.ItemView($FpageSize,$FOffset,[Microsoft.Exchange.WebServices.Data.OffsetBasePoint]::Beginning)
                                $itemView.Traversal = [Microsoft.Exchange.WebServices.Data.ItemTraversal]::Shallow
                                $itemView.PropertySet = new-object Microsoft.Exchange.WebServices.Data.PropertySet(
                                [Microsoft.Exchange.WebServices.Data.BasePropertySet]::IdOnly)
                                    $oFindFolders | %{
                                    $oFinditems = $exchangeService.FindItems($_.Id,$itemview)
                                        
                                    $obj = New-Object PSObject
                                    $obj | add-member noteproperty DisplayName $_.DisplayName
                                    $obj | add-member noteproperty PolicyTag (GetTagName $_.PolicyTag.RetentionId)
                                    $obj | add-member noteproperty ArchiveTag (GetTagName $_.ArchiveTag.RetentionId)                        
                                    $obj | add-member noteproperty RetentionActionPolicyTag (GetRetentionAction $_.PolicyTag.RetentionId)
                                    $obj | add-member noteproperty RetentionActionArchiveTag (GetRetentionAction $_.ArchiveTag.RetentionId)
                                    $obj | add-member noteproperty RetentionPeriodPolicyTag (GetRetentionPeriod $_.PolicyTag.RetentionId)
                                    $obj | add-member noteproperty RetentionPeriodArchiveTag (GetRetentionPeriod $_.ArchiveTag.RetentionId)
                                    $obj | add-member noteproperty Usermailbox $user
                                    $obj | add-member noteproperty FolderItemCount $oFinditems.TotalCount
                                    $Global:retentioninfo += $obj       
                                

                                }
                    

                            
                    }


                            

                            
                try{ 
                        $CurrentUser = get-recipient $user -ErrorAction STOP
                          
                                                 
                               
                        if(($CurrentUser).recipienttype -eq 'UserMailbox')
                        {
               
                            $exchangeService.Credentials = New-Object Microsoft.Exchange.WebServices.Data.WebCredentials -ArgumentList $credential.UserName, $credential.GetNetworkCredential().password
                            $id = New-Object Microsoft.Exchange.WebServices.Data.ImpersonatedUserId -ArgumentList "SmtpAddress",$user
                            $exchangeService.ImpersonatedUserId = $id

                            $exchangeService.AutodiscoverUrl($user)
             
                            folderretention;                        
                                         
                        }
                         
                        
                        
                        
                        
                        else{

                        
                        $exchangeService.Credentials = New-Object Microsoft.Exchange.WebServices.Data.WebCredentials -ArgumentList $o365credential.UserName, $o365credential.GetNetworkCredential().password
                        $id = New-Object Microsoft.Exchange.WebServices.Data.ImpersonatedUserId -ArgumentList "SmtpAddress",$user
                        $exchangeService.ImpersonatedUserId = $id
                        $exchangeService.Url = "https://outlook.office365.com/EWS/Exchange.asmx"

                        folderretention;

                        }  
                
                        }catch{
                            
                            $errcond = $_.Exception.Message
                            $timestamp = (get-date).DateTime
                            "Time of exception:  $timestamp" | Out-File $LogFile -Append
                            "User: $user" | out-file $LogFile -Append
                            $errcond | out-file -FilePath $LogFile -append
                        }

            
        
         

         
           
        }
        $Global:retentioninfo | Export-Csv report1.csv -NoTypeInformation
            
    }

        
           
      end{ 
      
      $endTime = Get-Date
      Write-Host "Total time = " + $($endTime -$StartTime)
      
      }       
       
               
        

    Tuesday, July 18, 2017 11:07 PM

Answers

  • Hi there - finally got this working - need to tidy it up a bit - moved the try - catch block outside the function - which uses get-recipient - the concept of the scope was tripping me up when it comes to runspaces - again thanks so much for the help
    • Marked as answer by nickkinn Saturday, July 22, 2017 10:32 AM
    Friday, July 21, 2017 1:02 PM

All replies

  • Generally you would handle performance as a multi step approach

    First step would be to optimize your code as much as possible EWS allows batching which is generally the first way of improving performance for multiple operations but from what I can see there are a few issues with your code first if there where more then 100 folders in the Mailbox you will only report on the first 100. You not checking if there are more folder available in further pages. Also why use FindItems if you just need the TotalItem count that can come from the Folder Object https://msdn.microsoft.com/en-us/library/microsoft.exchange.webservices.data.folder.totalcount%28v=exchg.80%29.aspx so that's a costly redundant call you making that isn't necessary if you not going to do item level reporting.

    Next step once the code is fully optimized would be looking to using multiple threads to process more the one mailbox simultaneously. If your not developer I wouldn't suggest that approach the easiest method is just run multiple version of the script with different inputs simultaneously. Even if you want one consolidated report its going to be much easier to just consolidate the different reports that the script generates then multi threading the process and dealing with the implications and issues that creates. Also if you are using Office365 you will potentially hit issues with throttling once you try to run this against multiple mailboxes simultaneously (if you started to multi thread then it would multiply this issue) that is something that you code will need to manage and respond to or it maybe be minimised using EWS impersonation. 

    Wednesday, July 19, 2017 2:38 AM
  • HI Glenn - excellent points as usual - i never realised about the TotalItem count on the folder object - also the paging - although i suspect in this environment a mailbox will not have more than 100 folders  - but it's possible. I read up a little bit about multithreading - and it is a complex topic - (much like EWS itself :)) so im running this script from my work desktop - connecting via remote powershell to one of the exchange servers - if i multithread - (95% of mailboxes are onprem) does it take into account the processor architecture on my own desktop - ie. number of cores etc. - or the server that i am connected to via powershell - sorry this is where it gets a bit confusing? Thanks for the valued assistance
    Wednesday, July 19, 2017 10:34 AM
  • Hi Glenn, thanks for that - I am already seeing a big increase in performance from previous - running against 4 mailboxes - 2onprem and 2 in the cloud taking 59 seconds - before it would take over 4 minutes.

    I just changed the pagesize to 500 - ie.   $FPageSize = 500  so I presume that means it will look up to 500 folders? chances of a mailbox exceeding that or extremely slim? or do you think it should check via .moreavailable = $true ?  Regards

    Wednesday, July 19, 2017 11:03 AM
  • >> if i multithread - (95% of mailboxes are onprem) does it take into account the processor architecture on my own desktop - ie. number of cores etc. - or the server that i am connected to via powershell

    Yes and No, but at that level its abstracted away eg your code will say create two threads and run X code on each thread and the O/S is responsible for scheduling and running that code on a particular processor and core but you don't need to worry about that because you can't control it. The Remote PowerShell calls are generated from your workstation  it doesn't take into account the Remote server (eg same as any call to a Web Server etc). The Exchange Server will throttle the Remote Powershell for that user if they exceed any of the throttling limits configured in the policy associated with the user you connecting with. If your running multiple threads this would mean multiple connections so there are concurrence limitation eg on Office365 its no more the 3 shells per user for Remote PowerShell and 37 concurrent EWS Connections (Onprem is generally 3 and 10 by default but you can change the policy OnPrem where in the cloud you generally can't). 

    >>I just changed the pagesize to 500 - ie.   $FPageSize = 500  so I presume that means it will look up to 500 folders? chances of a mailbox exceeding that or extremely slim? or do you think it should check via .moreavailable = $true ?  Regards

    The FindItem/FolderFolder throttling limit is 1000 so I always use 1000 and I've never had a problem. Yes you should check .moreavailable (and then increment the FolderView offset else you will create an infinite loop)  else you will miss those mailboxes where the folder count exceeds the page size your using.

    Wednesday, July 19, 2017 10:16 PM
  • Hi there, you say '37 concurrent EWS connections and 10 onprem' so having checked my laptop I have 4 logicalprocessors which means I can have a runspacepool with 4 threads - so therefore process 4 mailboxes simultaneously - thereby speeding up my script by an order of 4 making 4 concurrent EWS connections? not remote powershell connections?

    The other thing is I have created a script after reading up a bit on powershell multithreading but it just generates this error:  

    PS C:\Users\N0310355\desktop\scripts> .\run-multithread.ps1 -mbx (gc .\users.txt) -InputParam primarysmtpaddress
    Exception calling "EndInvoke" with "1" argument(s): "The pipeline has been stopped."
    At C:\Users\N0310355\desktop\scripts\run-multithread.ps1:50 char:13
    +             $results = $Job.Thread.EndInvoke($Job.Handle)
    +             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
        + CategoryInfo          : NotSpecified: (:) [], MethodInvocationException
        + FullyQualifiedErrorId : PipelineStoppedException

    Although to be honest I'm a bit confused as to what is going on or how to get this working:  It generates no output when I step through it with the powershell ISE Debugger - although for some reason in the debugger I can finish the script without the error above, from my reading this is the line where I should get data back:

    $Job.Thread.EndInvoke($Job.Handle) - so its the EndInvoke method which gives you your data back but I get nothing - and in my "Main EWS" script I am exporting the data to CSV, anyway here is what i came up with for the multithreading script: I read the EWS script file in as a scriptblock - but the script itself doesn't seem to execute even. Confused.




    Param(
        
        [Parameter(ValueFromPipeline=$true,ValueFromPipelineByPropertyName=$true)]$mbx,
        $InputParam = $Null
    )


    Begin{
      
        $OFS = "`r`n"
        $Code = [ScriptBlock]::Create($(Get-Content get-userretention.ps1))
        Remove-Variable OFS


        $rsPool = [runspacefactory]::CreateRunspacePool(1,4)
        $rsPool.Open()
        $jobs = @()
        $results = @()
    }



    process{

        foreach($mailbox in $mbx)
        {
            $PSinstance = [powershell]::Create().AddScript($Code)
            If ($InputParam -ne $Null){
                $PSinstance.AddParameter($InputParam, $mailbox.ToString()) | out-null}
            $PSinstance.RunspacePool = $rsPool
            $Handle = $PSinstance.BeginInvoke()
            $Job = "" | Select-Object Handle, Thread, object
            $Job.Handle = $Handle
            $Job.Thread = $PSinstance
            $Job.Object = $mailbox.ToString()
            $Jobs += $Job



        }
    }
    end{

        while(@($Jobs | Where-Object {$_.Handle -ne $Null}).count -gt 0)  {

        ForEach ($Job in $($Jobs | Where-Object {$_.Handle.IsCompleted -eq $True})){
                $Job.Thread.EndInvoke($Job.Handle)
                $Job.Thread.Dispose()
                $Job.Thread = $Null
                $Job.Handle = $Null
                
            }

        
        $rsPool.Close() | Out-Null
        $rsPool.Dispose() | Out-Null


        }
    }

    Wednesday, July 19, 2017 11:21 PM
  • >>so having checked my laptop I have 4 logicalprocessors which means I can have a runspacepool with 4 threads - so therefore process 4 mailboxes simultaneously - thereby speeding up my script by an order of 4 making 4 concurrent EWS connections? not remote powershell connections?

    No that's not correct your way off in thinking about how multi threading works you might want to research that more yourself, but lets say you only have one processor you can spawn as many threads as you like (or until the machine comes to as stop). The underling O/S will context switch those threads to run on the process while technically they don't run exactly at the same time modern processor can context switch so fast but you don't need to go into that really eg on your PC you will have many background threads running all you Apps you would just be adding to the pile of things to run. All you EWS requests are Web request so the threads will be in a wait state while the request is active and waiting for a response etc.

    IMO your trying to multi thread the wrong thing, eg let you Remote PowerShell call run in one thread and return the Mailboxes you want run the report on then invoke a separate job to run the EWS report on each Mailbox. Essentially one thread becomes the dispatcher which give you a level of control (eg don't invoke more the x number of jobs at once) otherwise your going to end potentially spawning 1000's of threads at once the way you trying to do it which may bring down the laptop or the server if you have throttling disabled. I'd suggest you have a look at Ingo's https://ingogegenwarth.wordpress.com/2015/04/16/get-mailbox-folder-permissions-using-ews-multithreading/ script which is basically doing what you want (but getting folder permission instead).

    Thursday, July 20, 2017 6:17 AM
  • Hi Glenn , thanks for that - I had a look at Ingo's script and tried to incorporate it into my own - since they are both 1) using ews and 2) using it against multiple mailboxes - so similar use case - however - mine still outputs nothing - I have a funny feeling its to do with variable scope or something because (without multithreading) i enumerate all the mailboxes  - collect them into a variable $Global:retentioninfo and output that to a CSV - and this all works fine - once i try to incorporate multithreading element into the script - I get nothing back






    [CmdletBinding()]

        param(

        [Parameter(
        Position = 0,
        Mandatory=$true,
        ValueFromPipelineByPropertyName=$true
        
        )]
        [String[]]
        $PrimarySmtpAddress,

        [parameter( Mandatory=$false, Position=1)]
    [ValidateRange(0,8)]
    [int]$Threads= '6',

        [parameter( Mandatory=$false, Position=2)]
    [switch]$MultiThread=$false,
        
        
    [parameter( Mandatory=$false, Position=3)]
    $MaxResultTime='240'

    )



    begin{
            
            
            $Jobs = @()
            $Sessionstate = [system.management.automation.runspaces.initialsessionstate]::CreateDefault()
            $RunspacePool = [RunspaceFactory]::CreateRunspacePool(1, $Threads,$Sessionstate, $Host)
            $RunspacePool.ApartmentState = "STA"
            $RunspacePool.Open()
            
             
       <#     
            #$startTime = Get-Date
            $error.clear()
            [string]$LogFile = "C:\Temp\Log.txt"  
            [int]$j='1'

            $o365credential = get-credential -credential o365creds
            
            $ExchangeSession = New-PSSession -ConfigurationName Microsoft.Exchange -ConnectionUri "https://outlook.office365.com/powershell-liveid/" -Credential $o365credential -Authentication "Basic" -AllowRedirection
            Import-PSSession $ExchangeSession -Prefix o365 -AllowClobber

            $credential = Get-Credential onpremcreds


            Add-Type -Path "C:\Program Files\Microsoft\Exchange\Web Services\2.0\Microsoft.Exchange.WebServices.dll"
            $exchangeService = New-Object Microsoft.Exchange.WebServices.Data.ExchangeService

            $Global:retentioninfo = @()
            $Global:RetentionTags = @()
            if($LogFile)
            {
                Remove-Item $LogFile -ea SilentlyContinue
            }
         #>   
    }



    process {
       
      
        function Get-UserFolderRentention{
        param(

        [Parameter(
        Position = 0,
        Mandatory=$true,
        ValueFromPipelineByPropertyName=$true
        
        )]
        [String[]]
        $PrimarySmtpAddress
        
        
        
        )

    $Global:retentioninfo = @()
            $Global:RetentionTags = @()



    #$startTime = Get-Date
            $error.clear()
            [string]$LogFile = "C:\Temp\Log.txt"  
            [int]$j='1'

            $o365credential = get-credential -credential o365creds
            
            $ExchangeSession = New-PSSession -ConfigurationName Microsoft.Exchange -ConnectionUri "https://outlook.office365.com/powershell-liveid/" -Credential $o365credential -Authentication "Basic" -AllowRedirection
            Import-PSSession $ExchangeSession -Prefix o365 -AllowClobber

            $credential = Get-Credential "onpremcreds"


            Add-Type -Path "C:\Program Files\Microsoft\Exchange\Web Services\2.0\Microsoft.Exchange.WebServices.dll"
            $exchangeService = New-Object Microsoft.Exchange.WebServices.Data.ExchangeService

            $Global:retentioninfo = @()
            $Global:RetentionTags = @()
            if($LogFile)
            {
                Remove-Item $LogFile -ea SilentlyContinue
            }
                 
                   function folderretention()
                   {
                            Write-Host -ForegroundColor DarkGreen "Hello inside function"

                            $FPageSize = 500
                            $FOffset = 0
                            $MoreItems =$true
                            $ItemCount=0
                            $folderView = new-object Microsoft.Exchange.WebServices.Data.FolderView($FPageSize,$FOffset,[Microsoft.Exchange.WebServices.Data.OffsetBasePoint]::Beginning)
                            $folderView.Traversal = [Microsoft.Exchange.WebServices.Data.FolderTraversal]::Deep
                            $oFindFolders = $exchangeService.FindFolders([Microsoft.Exchange.WebServices.Data.WellKnownFolderName]::MsgFolderRoot,$null,$folderView)
                            #?{(($_.DisplayName -notlike 'Calendar') -and ($_.DisplayName -notlike '*contacts') -and ($_.DisplayName -notlike '*recipient*'))}
                
                            $Global:RetentionTags = $exchangeService.GetUserRetentionPolicyTags()
                    

                                function GetTagName($tagGUID) {
                                if (!$tagGUID) { return ($RetentionTags.RetentionPolicyTags | ? {$_.Type -eq "All"}).DisplayName }
                                foreach ($tag in $RetentionTags.RetentionPolicyTags) {
                                if ($tag.RetentionId -eq $tagGUID ) { return $tag.DisplayName }
                                }
                            }


                                function GetRetentionAction($tagGUID)  {
                                if (!$tagGUID) { return ($RetentionTags.RetentionPolicyTags | ? {$_.Type -eq "All"}).RetentionAction }
                                foreach ($tag in $RetentionTags.RetentionPolicyTags) {
                                if ($tag.RetentionId -eq $tagGUID ) { return $tag.RetentionAction }

                                }
                            }

                                 function GetRetentionPeriod($tagGUID)  {
                                if (!$tagGUID) { return ($RetentionTags.RetentionPolicyTags | ? {$_.Type -eq "All"}).RetentionPeriod }
                                foreach ($tag in $RetentionTags.RetentionPolicyTags) {
                                if ($tag.RetentionId -eq $tagGUID ) { return $tag.RetentionPeriod }

                                }
                            }
                                
                           
                           
                               # $itemView = new-object Microsoft.Exchange.WebServices.Data.ItemView($FpageSize,$FOffset,[Microsoft.Exchange.WebServices.Data.OffsetBasePoint]::Beginning)
                               # $itemView.Traversal = [Microsoft.Exchange.WebServices.Data.ItemTraversal]::Shallow
                               # $itemView.PropertySet = new-object Microsoft.Exchange.WebServices.Data.PropertySet(
                                #[Microsoft.Exchange.WebServices.Data.BasePropertySet]::IdOnly)
                                    $oFindFolders | %{
                                  #  $oFinditems = $exchangeService.FindItems($_.Id,$itemview)
                                        
                                    $obj = New-Object PSObject
                                    $obj | add-member noteproperty DisplayName $_.DisplayName
                                    $obj | add-member noteproperty PolicyTag (GetTagName $_.PolicyTag.RetentionId)
                                    $obj | add-member noteproperty ArchiveTag (GetTagName $_.ArchiveTag.RetentionId)                        
                                    $obj | add-member noteproperty RetentionActionPolicyTag (GetRetentionAction $_.PolicyTag.RetentionId)
                                    $obj | add-member noteproperty RetentionActionArchiveTag (GetRetentionAction $_.ArchiveTag.RetentionId)
                                    $obj | add-member noteproperty RetentionPeriodPolicyTag (GetRetentionPeriod $_.PolicyTag.RetentionId)
                                    $obj | add-member noteproperty RetentionPeriodArchiveTag (GetRetentionPeriod $_.ArchiveTag.RetentionId)
                                    $obj | add-member noteproperty Usermailbox $user
                                    $obj | add-member noteproperty FolderItemCount $_.TotalCount
                                    $Global:retentioninfo += $obj       
                                

                                }
                    
    $Global:retentioninfo
                            
                    }


                            

                            
                try{ 
                        $CurrentUser = get-recipient $user -ErrorAction STOP
                          
                                                 
                               
                        if(($CurrentUser).recipienttype -eq 'UserMailbox')
                        {
               
                            $exchangeService.Credentials = New-Object Microsoft.Exchange.WebServices.Data.WebCredentials -ArgumentList $credential.UserName, $credential.GetNetworkCredential().password
                            $id = New-Object Microsoft.Exchange.WebServices.Data.ImpersonatedUserId -ArgumentList "SmtpAddress",$user
                            $exchangeService.ImpersonatedUserId = $id

                            $exchangeService.Url = "https://onpremserver/ews/exchange.asmx"
             
                            folderretention;                        
                                         
                        }
                         
                        
                        
                        
                        
                        else{

                        
                        $exchangeService.Credentials = New-Object Microsoft.Exchange.WebServices.Data.WebCredentials -ArgumentList $o365credential.UserName, $o365credential.GetNetworkCredential().password
                        $id = New-Object Microsoft.Exchange.WebServices.Data.ImpersonatedUserId -ArgumentList "SmtpAddress",$user
                        $exchangeService.ImpersonatedUserId = $id
                        $exchangeService.Url = "https://outlook.office365.com/EWS/Exchange.asmx"

                        folderretention;

                        }  
                
                        }catch{
                            
                            $errcond = $_.Exception.Message
                            $timestamp = (get-date).DateTime
                            "Time of exception:  $timestamp" | Out-File $LogFile -Append
                            "User: $user" | out-file $LogFile -Append
                            $errcond | out-file -FilePath $LogFile -append
                        }

            
        
         

         
           
        }
           

    If($MultiThread) {
    #create scriptblock from function
    $ScriptBlock = [scriptblock]::Create((Get-ChildItem Function:\Get-UserFolderRentention).Definition)
        ForEach($Address in $Primarysmtpaddress) {
            try{
    $j++ | Out-Null
                $MailboxName = $Address
                $PowershellThread = [powershell]::Create().AddScript($ScriptBlock).AddParameter('Primarysmtpaddress',$MailboxName)
                
                $PowershellThread.RunspacePool = $RunspacePool
    $Handle = $PowershellThread.BeginInvoke()
    $Job = "" | Select-Object Handle, Thread, object
    $Job.Handle = $Handle
    $Job.Thread = $PowershellThread
    $Job.Object = $Address #.ToString()
    $Jobs += $Job
                }
                catch{
                    $Error[0].Exception
                }
            }      
        }
    }

        
           
      end{ 
          If ($MultiThread) {
       $SleepTimer = 200
       $ResultTimer = Get-Date
       While (@($Jobs | Where-Object {$_.Handle -ne $Null}).count -gt 0)  {
       $Remaining = "$($($Jobs | Where-Object {$_.Handle.IsCompleted -eq $False}).object)"
       If ($Remaining.Length -gt 60){
       $Remaining = $Remaining.Substring(0,60) + "..."
       }
       Write-Progress `
       -id 1 `
       -Activity "Waiting for Jobs - $($Threads - $($RunspacePool.GetAvailableRunspaces())) of $Threads threads running" `
       -PercentComplete (($Jobs.count - $($($Jobs | Where-Object {$_.Handle.IsCompleted -eq $False}).count)) / $Jobs.Count * 100) `
       -Status "$(@($($Jobs | Where-Object {$_.Handle.IsCompleted -eq $False})).count) remaining - $Remaining"

       ForEach ($Job in $($Jobs | Where-Object {$_.Handle.IsCompleted -eq $True})){
       $Job.Thread.EndInvoke($Job.Handle)
       $Job.Thread.Dispose()
       $Job.Thread = $Null
       $Job.Handle = $Null
       $ResultTimer = Get-Date
       }
       If (($(Get-Date) - $ResultTimer).totalseconds -gt $MaxResultTime){
       Write-Warning "Child script appears to be frozen for $($Job.Object), try increasing MaxResultTime"
       #Exit
       }
       Start-Sleep -Milliseconds $SleepTimer
       # kill all incomplete threads when hit "CTRL+q"
       If ($Host.UI.RawUI.KeyAvailable) {
       $KeyInput = $Host.UI.RawUI.ReadKey("IncludeKeyUp,NoEcho")
       If (($KeyInput.ControlKeyState -cmatch '(Right|Left)CtrlPressed') -and ($KeyInput.VirtualKeyCode -eq '81')) {
       Write-Host -fore red "Kill all incomplete threads....."
       ForEach ($Job in $($Jobs | Where-Object {$_.Handle.IsCompleted -eq $False})){
       Write-Host -fore yellow "Stopping job $($Job.Object) ...."
       $Job.Thread.Stop()
       $Job.Thread.Dispose()
       }
       Write-Host -fore red "Exit script now!"
       Exit
       }
       }
       }
       # clean-up
       $RunspacePool.Close() | Out-Null
       $RunspacePool.Dispose() | Out-Null
       [System.GC]::Collect()
    }
    }


       
               
        

    Thursday, July 20, 2017 12:42 PM
  • Your global variable won't be in scope across thread boundaries, I would suggest you either write a separate csv for each job then consolidate them together at the end using Import-csv or you should be able to return the results as an Object from each job and then consolidate that from the calling Main thread.
    Thursday, July 20, 2017 10:00 PM
  • Hi Glenn

    I've changed the script around - and I feel I am very close - but this is what I am seeing now in the error log I have - I know its executing the function and taking in the arguments - its at this block its failing

    try{ 
                        write-host "in the function"
                        $CurrentUser = get-recipient $user -ErrorAction STOP

    and the error I get in the log is :   

    Time of exception:  21 July 2017 13:06:58
    User: nicholas.herbert@domain.com
    The term 'get-recipient' is not recognized as the name of a cmdlet, function, script file, or operable program. Check the spelling of the name, or if a path was included, verify that the path is correct and try again.

    It doesnt seem to recognise the exchange cmdlets from with the function - and I need these to determine if the user is onprem or in the cloud?  Here is the refactored script - annoying thing about the multithreading idea - is that you cant debug the script/function while it is running inside the thread/runspacepool - so your kinda blind

    Regards






    [CmdletBinding()]

        param(

        [Parameter(
        Position = 0,
        Mandatory=$true,
        ValueFromPipelineByPropertyName=$true
        
        )]
        [String[]]
        $PrimarySmtpAddress,

        [parameter( Mandatory=$false, Position=1)]
    [switch]$MultiThread=$false

        

    )



    begin{
    <#
        if (-not (Get-PSSnapin | where {$_.Name -eq "Microsoft.Exchange.Management.PowerShell.SnapIn"} )) {
            Write-Host "Adding Exchange 2013 PowerShell role"
            Add-PSSnapin Microsoft.Exchange.Management.PowerShell.SnapIn

            }

      #>    $Jobs = @()
            $Sessionstate = [system.management.automation.runspaces.initialsessionstate]::CreateDefault()
            $RunspacePool = [RunspaceFactory]::CreateRunspacePool(1, 5,$Sessionstate, $Host)
            $RunspacePool.ApartmentState = "STA"
            $RunspacePool.Open()
            
            $startTime = Get-Date
            $error.clear()
            [string]$LogFile = "C:\Temp\Log.txt"  
            
            $o365credential = get-credential -credential o365creds
            

            $ExchangeSession = New-PSSession -ConfigurationName Microsoft.Exchange -ConnectionUri "https://outlook.office365.com/powershell-liveid/" -Credential $o365credential -Authentication "Basic" -AllowRedirection
            Import-PSSession $ExchangeSession -Prefix o365 -AllowClobber

            


      #      Add-Type -Path "C:\Program Files\Microsoft\Exchange\Web Services\2.0\Microsoft.Exchange.WebServices.dll"
      #      $exchangeService = New-Object Microsoft.Exchange.WebServices.Data.ExchangeService

          $retentioninfo = @()
          $RetentionTags = @()
           
    }



    process {
       
      
      
      function Get-MailboxFoldertags()
      {
           param(


               [Parameter(ValueFromPipelineByPropertyName=$true)]
                [string]
                $user,
               
                $o365credential
              

           )  
           
            $retentioninfo = @()
            $retentiontags = @()
            [string]$LogFile = "C:\Temp\Log.txt"
             if($LogFile)
            {
                Remove-Item $LogFile -ea SilentlyContinue
            }


            Add-Type -Path "C:\Program Files\Microsoft\Exchange\Web Services\2.0\Microsoft.Exchange.WebServices.dll"
            $exchangeService = New-Object Microsoft.Exchange.WebServices.Data.ExchangeService
           
               
                   function folderretention()
                   {


                            
                            $FPageSize = 500
                            $FOffset = 0
                           
                            $folderView = new-object Microsoft.Exchange.WebServices.Data.FolderView($FPageSize,$FOffset,[Microsoft.Exchange.WebServices.Data.OffsetBasePoint]::Beginning)
                            $folderView.Traversal = [Microsoft.Exchange.WebServices.Data.FolderTraversal]::Deep
                            $oFindFolders = $exchangeService.FindFolders([Microsoft.Exchange.WebServices.Data.WellKnownFolderName]::MsgFolderRoot,$null,$folderView)
                            #?{(($_.DisplayName -notlike 'Calendar') -and ($_.DisplayName -notlike '*contacts') -and ($_.DisplayName -notlike '*recipient*'))}
                
                            $RetentionTags = $exchangeService.GetUserRetentionPolicyTags()
                    

                                function GetTagName($tagGUID) {
                                if (!$tagGUID) { return ($RetentionTags.RetentionPolicyTags | ? {$_.Type -eq "All"}).DisplayName }
                                foreach ($tag in $RetentionTags.RetentionPolicyTags) {
                                if ($tag.RetentionId -eq $tagGUID ) { return $tag.DisplayName }
                                }
                            }


                                function GetRetentionAction($tagGUID)  {
                                if (!$tagGUID) { return ($RetentionTags.RetentionPolicyTags | ? {$_.Type -eq "All"}).RetentionAction }
                                foreach ($tag in $RetentionTags.RetentionPolicyTags) {
                                if ($tag.RetentionId -eq $tagGUID ) { return $tag.RetentionAction }

                                }
                            }

                                 function GetRetentionPeriod($tagGUID)  {
                                if (!$tagGUID) { return ($RetentionTags.RetentionPolicyTags | ? {$_.Type -eq "All"}).RetentionPeriod }
                                foreach ($tag in $RetentionTags.RetentionPolicyTags) {
                                if ($tag.RetentionId -eq $tagGUID ) { return $tag.RetentionPeriod }

                                }
                            }
                                
                           
                           
                               # $itemView = new-object Microsoft.Exchange.WebServices.Data.ItemView($FpageSize,$FOffset,[Microsoft.Exchange.WebServices.Data.OffsetBasePoint]::Beginning)
                               # $itemView.Traversal = [Microsoft.Exchange.WebServices.Data.ItemTraversal]::Shallow
                               # $itemView.PropertySet = new-object Microsoft.Exchange.WebServices.Data.PropertySet(
                                #[Microsoft.Exchange.WebServices.Data.BasePropertySet]::IdOnly)
                                    $oFindFolders | %{
                                  #  $oFinditems = $exchangeService.FindItems($_.Id,$itemview)
                                        
                                    $obj = New-Object PSObject
                                    $obj | add-member noteproperty DisplayName $_.DisplayName
                                    $obj | add-member noteproperty PolicyTag (GetTagName $_.PolicyTag.RetentionId)
                                    $obj | add-member noteproperty ArchiveTag (GetTagName $_.ArchiveTag.RetentionId)                        
                                    $obj | add-member noteproperty RetentionActionPolicyTag (GetRetentionAction $_.PolicyTag.RetentionId)
                                    $obj | add-member noteproperty RetentionActionArchiveTag (GetRetentionAction $_.ArchiveTag.RetentionId)
                                    $obj | add-member noteproperty RetentionPeriodPolicyTag (GetRetentionPeriod $_.PolicyTag.RetentionId)
                                    $obj | add-member noteproperty RetentionPeriodArchiveTag (GetRetentionPeriod $_.ArchiveTag.RetentionId)
                                    $obj | add-member noteproperty Usermailbox $user
                                    $obj | add-member noteproperty FolderItemCount $_.TotalCount
                                    $retentioninfo += $obj       
                                

                                }
                    

                            
                    }


                            

                            
                try{ 
                        write-host "in the function"
                        $CurrentUser = get-recipient $user -ErrorAction STOP
                         write-host -ForegroundColor DarkGreen "still in function"
                                                 
                               
                        if(($CurrentUser).recipienttype -eq 'UserMailbox')
                        {
                            $exchangeService.UseDefaultCredentials =$true
                            #$exchangeService.Credentials = New-Object Microsoft.Exchange.WebServices.Data.WebCredentials -ArgumentList $credential.UserName, $credential.GetNetworkCredential().password
                            $id = New-Object Microsoft.Exchange.WebServices.Data.ImpersonatedUserId -ArgumentList "SmtpAddress",$user
                            $exchangeService.ImpersonatedUserId = $id

                            $exchangeService.AutodiscoverUrl($user)
                            Write-Host -ForegroundColor DarkGreen "test"
                            folderretention;                        
                                         
                        }
                         
                        
                        
                        
                        
                        else{

                        
                        $exchangeService.Credentials = New-Object Microsoft.Exchange.WebServices.Data.WebCredentials -ArgumentList $o365credential.UserName, $o365credential.GetNetworkCredential().password
                        $id = New-Object Microsoft.Exchange.WebServices.Data.ImpersonatedUserId -ArgumentList "SmtpAddress",$user
                        $exchangeService.ImpersonatedUserId = $id
                        $exchangeService.Url = "https://outlook.office365.com/EWS/Exchange.asmx"

                        folderretention;

                        }  
                
                        }catch{
                            
                            $errcond = $_.Exception.Message
                            $timestamp = (get-date).DateTime
                            "Time of exception:  $timestamp" | Out-File $LogFile -Append
                            "User: $user" | out-file $LogFile -Append
                            $errcond | out-file -FilePath $LogFile -append
                        }

            
        
         

         
           
        
        #$retentioninfo
     }
        if($MultiThread)
        {

            $ScriptBlock = [scriptblock]::Create((Get-ChildItem Function:\Get-MailboxFoldertags).Definition)
            
            foreach($user in $PrimarySmtpAddress){
              $PowershellThread = [powershell]::Create().AddScript($ScriptBlock)
              $PowershellThread.AddArgument($user) | Out-Null
             
              $PowershellThread.AddArgument($o365credential) | Out-Null
             
              $PowershellThread.RunspacePool = $RunspacePool
    $Handle = $PowershellThread.BeginInvoke()
    $Job = "" | Select-Object Handle, Thread, object
    $Job.Handle = $Handle
    $Job.Thread = $PowershellThread
    $Job.Object = $user #.ToString()
    $Jobs += $Job
              
                
            }
        
        }    
    }

        
           
      end{ 
        
            if($MultiThread)
            {
                While (@($Jobs | Where-Object {$_.Handle -ne $Null}).count -gt 0)  {
                   ForEach ($Job in $($Jobs | Where-Object {$_.Handle.IsCompleted -eq $True})){
       $Job.Thread.EndInvoke($Job.Handle)
       $Job.Thread.Dispose()
       $Job.Thread = $Null
       $Job.Handle = $Null

           }   

                }
                $RunspacePool.Close() | Out-Null
           $RunspacePool.Dispose() | Out-Null
            }



      #$endTime = Get-Date
      #Write-Host "Total time = " + $($endTime -$StartTime)
      
      }       
       
               
        

    Friday, July 21, 2017 12:22 PM
  • Hi there - finally got this working - need to tidy it up a bit - moved the try - catch block outside the function - which uses get-recipient - the concept of the scope was tripping me up when it comes to runspaces - again thanks so much for the help
    • Marked as answer by nickkinn Saturday, July 22, 2017 10:32 AM
    Friday, July 21, 2017 1:02 PM