PowerShell script to split large text files into smaller files by line count, efficiently handling big data without high memory usage and creating ordered outpu
# Script to split a large text file into smaller files
param(
[Parameter(Mandatory=$true)]
[string]$FileInputPath, # Full path of the text file to split
[Parameter(Mandatory=$true)]
[int]$MaxRighePerFile, # Maximum number of lines per output file
[Parameter(Mandatory=$false)]
[string]$CartellaOutput = "Output" # Folder to save the split files (default: ./Output)
)
# Check that the input file exists
if (-not (Test-Path -Path $FileInputPath)) {
Write-Error "The input file '$FileInputPath' does not exist."
exit 1
}
# Create the output folder if it does not exist
if (-not (Test-Path -Path $CartellaOutput)) {
New-Item -ItemType Directory -Path $CartellaOutput | Out-Null
}
# Extract the base file name without extension
$NomeBaseFile = [System.IO.Path]::GetFileNameWithoutExtension($FileInputPath)
# Extract the file extension
$EstensioneFile = [System.IO.Path]::GetExtension($FileInputPath)
# Initialize counters
$ContatoreFile = 1
$ContatoreRighe = 0
# Build the output file name
$NomeFileOutput = "{0}_{1}{2}" -f $NomeBaseFile, $ContatoreFile, $EstensioneFile
$PercorsoFileOutput = Join-Path -Path $CartellaOutput -ChildPath $NomeFileOutput
# Open the output file for writing
$StreamWriter = [System.IO.StreamWriter]::new($PercorsoFileOutput, $false)
try {
# Read the input file line by line
Get-Content -Path $FileInputPath -ReadCount 1 | ForEach-Object {
$riga = $_
# Write the line to the current output file
$StreamWriter.WriteLine($riga)
$ContatoreRighe++
# If the maximum number of lines is reached, close the file and open a new one
if ($ContatoreRighe -ge $MaxRighePerFile) {
$StreamWriter.Close()
$ContatoreFile++
$ContatoreRighe = 0
$NomeFileOutput = "{0}_{1}{2}" -f $NomeBaseFile, $ContatoreFile, $EstensioneFile
$PercorsoFileOutput = Join-Path -Path $CartellaOutput -ChildPath $NomeFileOutput
$StreamWriter = [System.IO.StreamWriter]::new($PercorsoFileOutput, $false)
}
}
}
finally {
# Ensure the stream is closed even in case of error
if ($StreamWriter -ne $null) {
$StreamWriter.Close()
}
}
Write-Host "Splitting completed. Files created in folder: $CartellaOutput"PowerShell 5.1 or higher
Input text file (.txt)
DEPENDENCIES: no external, uses only standard PowerShell cmdlets
CONFIG: input parameters - source file path, maximum number of lines per output file, destination folder for split files