とりあえずこうすると回避できた。

using namespace Microsoft.Office.Interop.Excel

$excel = New-Object -ComObject Excel.Application
$excel.Visible = $True
$fileName = (Get-ChildItem "test.xlsx").FullName

$param = @([Type]::Missing) * 15
$param[0] = $fileName
$param[14] = [XlCorruptLoad]::xlRepairFile
$workbooks = $excel.Workbooks
$book = $workbooks.Open.Invoke($param)