I had a rather big file with 35000 items in it. I wanted to take a phased approach and found a script to split the CSV file based on a number of lines per file. Script Center repository:”http://gallery.technet.microsoft.com/scriptcenter/PowerShell-Split-large-log-6f2c4da0
I tweaked the script a little bit, so only the parameters that are necessary are File Name and Number of Lines Per File.
$linecount = 0
$filenumber = 1
$sourcefilename = Read-Host "What is the full path and name of the log file to split? (e.g. D:\mylogfiles\mylog.txt) "
$destinationfolderpath = Split-Path $sourcefilename -parent
$srcfile = gci $sourcefilename
$filebasename = $srcfile.BaseName
$fileext = $srcfile.Extension
Get-Content $sourcefilename | Measure-Object | ForEach-Object { $sourcelinecount = $_.Count }
Write-Host "Your current file size is $sourcelinecount lines long"
$destinationfilesize = Read-Host "How many lines will be in each new split file? "
$maxsize = [int]$destinationfilesize
Write-Host File is $sourcefilename - destination is $destinationfolderpath - new file line count will be $destinationfilesize
Write-Host "Writing part: $destinationfolderpath$filebasename`_part$filenumber$fileext"
$content = get-content $sourcefilename | % {
#Add-Content $destinationfolderpath$filebasename_$filenumber.txt "$_"
Add-Content $destinationfolderpath$filebasename`_part$filenumber$fileext "$_"
$linecount ++
If ($linecount -eq $maxsize) {
$filenumber++
$linecount = 0
Write-Host "Writing part: $destinationfolderpath$filebasename`_part$filenumber$fileext"
}
}
|