newswan

@echo off
setlocal ENABLEDELAYEDEXPANSION
chcp 936

set path=%pathgnu%;%path%

set sour=新建文件夹
set dest=%~n0
set exclude=QQ

if exist %dest% rd /s /q %dest%
if not exist %dest% mkdir %dest%

call :C_timer

set "_cmd_=dir /s /b %sour%\*.txt | sed -r -e "s/^.+[\]//" | sort -u"
set _cmdsed_=-r -e "/%exclude%/d" -e "/^\S+\s+\S+\s+\S+$/^!d" -e "s/SH/0/" -e "s/SZ/1/" -e "s/\s+/^|/g"
set _cmdawk_=" { arr[$0]++ ; if ( arr[$0] == 1 ) { print $0 } } "
for /f "usebackq tokens=* delims=" %%a in (`cmd /c "%_cmd_%"`) do (
    echo %%a
    (
        for /f "usebackq tokens=* delims=" %%b in (`dir /s /b "%sour%\%%a"`) do (
            type "%%~fb"
        ) | sed -r -e $a\
    ) | sed !_cmdsed_! | awk !_cmdawk_! > "%dest%\%%~nxa"
)

call :C_timer sum

exit/b

rem del %dest%\*.tmp


:C_timer
if not defined _ti_ set _ti_=-1
set/a _ti_+=1
set _tb_=%_te_%
set _te_=%time%
if %_ti_% EQU 0 (
    set _tb_=%_te_%
)
set/a _tdiff_=(9%_te_:~0,2%-9%_tb_:~0,2%)*360000+(9%_te_:~3,2%-9%_tb_:~3,2%)*6000+(9%_te_:~6,2%%_te_:~9,2%-9%_tb_:~6,2%%_tb_:~9,2%)
set/a _tdiff_=%_tdiff_:-=8640000-%
set/a _tdiffSum_+=%_tdiff_%
if "%_ti_%" == "0" (
    (echo,) >>time.txt
    (echo %~nx0  %_ti_%  %_te_%) >>time.txt
) else (
    (echo %~nx0  %_ti_%  %_te_%  %_tdiff_:~0,-2%.%_tdiff_:~-2%) >>time.txt
)
if %1.==sum. (
    (echo %~nx0                  %_tdiffSum_:~0,-2%.%_tdiffSum_:~-2%) >>time.txt
)
goto :eof
复制代码

newswan

回复 52# xczxczxcz


可以在vbs里加个计时器，完成后显示时长
这样时间比较准确

xczxczxcz

原VSCODE调试50秒，
今优化了两点：I/O读取和范型数据处理，现缩短至平均24秒(23.7XX~24.2XX)。（按顶楼的要求和网盘文件）

视频被百度转码变成卡顿了

第一个视频为前台处理：约24秒
https://pan.baidu.com/s/1VXnS_n6mE457iUp-rLk8Hg

第二个视频为后台处理：约28秒
https://pan.baidu.com/s/1S6Dl1KqAus_QpBVkTcOYWw

newswan

同名文件分组，流式处理
需要 sed awk sort

@echo off
setlocal ENABLEDELAYEDEXPANSION

set sour=新建文件夹
set dest=%~n0
set exclude=QQ

set "_cmd_=dir /s /b %sour%\*.txt | sed -r -e "s/^.+[\]//" | sort -u"
set _cmdsed_=-r -e "/%exclude%/d" -e "/^\S+\s+\S+\s+\S+$/^!d" -e "s/SZ/1/" -e "s/SH/0/" -e "s/\s+/^|/g"
set _cmdawk_=" { arr[$0]++ ; if ( arr[$0] == 1 ) { print $0 } } "
for /f "usebackq tokens=* delims=" %%a in (`cmd /c "%_cmd_%"`) do (
    echo %%a
    (
        for /f "usebackq tokens=* delims=" %%b in (`dir /s /b "%sour%\%%a"`) do (
            type "%%~fb"
        ) | sed -r -e $a\
    ) | sed !_cmdsed_! | awk !_cmdawk_! > "%dest%\%%~nxa"
)
复制代码

ps: linux 的 sort 不是 windows自带的

WHY

Test.vbs

Rem On Error Resume Next
Dim srcDir, dstDir

srcDir = "E:\Test\新建文件夹"           '源文件夹路径，需替换为实际路径
dstDir = "E:\Test\新建文件夹1"          '目标文件夹路径

Dim wsc
Set wsc = CreateObject("WScript.Shell")

If WSH.Arguments.Count = 0 Then
    wsc.Run "cscript " & chr(34) & WSH.ScriptFullName & chr(34) & " ARG", 0
    WScript.Quit
End If

Dim dic1, fso
Set dic1 = CreateObject("Scripting.Dictionary")
Set fso  = CreateObject("Scripting.FileSystemObject")

Dim reg
Set reg     = New RegExp
reg.Pattern = "[ \t]+"
reg.Global  = True

Dim objExec
If Not fso.FolderExists(srcDir) Then MsgBox "源文件夹不存在" : WSH.Quit
wsc.Run "cmd /c md " & chr(34) & dstDir & chr(34) & " 2>nul", 0              '创建目标文件夹
Set objExec = wsc.Exec( "cmd /c dir /b /a-d /s " & chr(34) & srcDir & "\*.txt" & chr(34) )  '遍历源文件夹

Rem 字典dic1，key=文件名，value=文件路径
Dim strFilePath, strFileName
Do Until objExec.StdOut.AtEndOfStream
    strFilePath = objExec.StdOut.ReadLine
    strFileName = fso.GetFile(strFilePath).Name
    strFileName = LCase(strFileName)
    If Not dic1.Exists(strFileName) Then
        dic1.Add strFileName, strFilePath
    Else
        dic1.Item(strFileName) = dic1.Item(strFileName) & vbLf & strFilePath
    End If
Loop

Rem 遍历字典dic1
Dim key
For Each key In dic1.Keys 
    GetFileContent key, Split( Dic1.Item(key), vbLf )
Next

Function GetFileContent(fileName, ByRef arrFilePath)
    Dim dic2, i, objFile, strLine
    Set dic2 = CreateObject("Scripting.Dictionary")

    For i = 0 To UBound(arrFilePath)
        Set objFile = fso.OpenTextFile(arrFilePath(i), 1)
        Do Until objFile.AtEndOfStream
            strLine = objFile.ReadLine
            strLine = reg.Replace(strLine, "|")                                       '空格替换为"|"
            If UBound(Split(strLine, "|")) >= 2 And InStr(strLine, "-QQ") <= 0 Then   '如果大于等于3列、不含 "-QQ"
                strLine = Replace(Replace(strLine,"SZ", "0|"),"SH", "1|")             '替换 "SZ" 和 "SH"
                If Not dic2.Exists(strLine) Then                                      '字典dic2，用于去重复
                    dic2.Add strLine, True
                End If
            End If 
        Loop
        objFile.Close
        Set objFile = Nothing
    Next

    fso.OpenTextFile(dstDir & "\" & fileName, 2, True).Write( Join(dic2.Keys, vbCrLf) )    '保存
    Set dic2 = Nothing
End Function

MsgBox "Done"
复制代码

Test.PS1

$dt = get-Date;

$srcDir = 'E:\Test\新建文件夹';
$dstDir = 'E:\Test\新建文件夹1';

If (![IO.Directory]::Exists($dstDir)){ $null = md $dstDir }

$dic = New-Object 'System.Collections.Generic.Dictionary[string, [Collections.ArrayList]]';

forEach( $file In (dir $srcDir -Recurse -Filter '*.txt') ){
    $Name = $file.BaseName.ToUpper();
    $Path = $file.FullName;
    if( !$dic.ContainsKey($Name) ){
        $dic.Add($Name, @($Path));
    } else {
        [void]$dic[$Name].Add($Path);
    }
}

forEach( $key In $dic.Keys ){
    $Hash = @{};
    for( $i=0; $i -lt $dic[$key].Count; $i++ ){
        $arr = [IO.File]::ReadAllLines($dic[$key][$i], [Text.Encoding]::Default);
        $arr = $arr -match '^(?:(?!-QQ)\S)+\s+\S+\s+\S+';
        $arr = $arr -replace '\s+', '|' -replace 'SZ', '0|' -replace 'SH', '1|';
        $Count = $arr.Count;
        for( $j=0; $j -lt $Count; $j++ ){
            if( !$Hash.ContainsKey($arr[$j]) ){
                $Hash.Add($arr[$j], $True);
            }
        }
    }
    [IO.File]::WriteAllLines( $dstDir + '\' + $key + '.txt', $Hash.Keys );
}
((get-Date) - $dt).TotalSeconds;
复制代码

newswan

相当于

 $null = 
复制代码

went

回复 47# idwma


     估计是[void]直接强转成了$null才不会输出

idwma

回复 44# newswan


    看went的代码发现一个[void],把空输出改成这个瞬间起飞呀

Get-ChildItem -path $dest *.tmp | foreach-object {
    $reader = New-Object -TypeName System.IO.StreamReader -ArgumentList $_.fullname
    $aa=New-Object System.Collections.Generic.HashSet[string]
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".txt")
    while ( $read = $reader.ReadLine() ) {
        [void]$aa.add($read)
    }
    [IO.File]::WriteAllLines($filename,$aa)
}
复制代码

went

回复 45# went


    线程不要设置过多,不然cpu容易爆满

went

可以尝试下多线程,线程数越多越快,取决你的cpu
数据处理函数可以自行优化

#&cls&cd /d "%~dp0" & @powershell -c "Get-Content '%~0' | Select-Object -Skip 1 | Out-String | Invoke-Expression" & pause&exit
cls
$t1 = Get-Date
$src_dir = '新建文件夹'
$dst_dir = 'out'

[void][System.IO.Directory]::CreateDirectory($dst_dir)

#线程函数 处理数据
$HandleGroupJob = {
    #线程参数
    param($dst_dir,$groupInfo)
    Write-Host $groupInfo.Name
    #汇总去重并筛选
	$set = New-Object 'System.Collections.Generic.HashSet[string]'
    & { $groupInfo.Group | foreach { [IO.File]::ReadAllLines($_.FullName)} } | foreach {
		if($_ -match '\S+\s+\d+\s+-?\d{2,}'){
			[void]$set.Add(($_ -replace 'SH','1|' -replace 'SZ','0|' -replace '\s+','|'))
		}
	}
    #输出
    Out-File -InputObject $set -FilePath ('{0}\{1}' -f $dst_dir,$groupInfo.Name) 
	$set = $null
    return ($groupInfo.Name + ' 已完成')
}

#多线程设置
$pool = [runspacefactory]::CreateRunspacePool(1,10) #最多10个线程并发
$pool.Open()
$threads = New-Object 'System.Collections.ArrayList'
$results = New-Object 'System.Collections.ArrayList'

'开始创建线程...'
Get-ChildItem -Recurse -Path $src_dir -Filter '*.txt' | Group-Object {$_.Name} | foreach {
    $_.Name
    $thread = [powershell]::Create()
    $thread.RunspacePool = $pool
    [void]$thread.AddScript($HandleGroupJob)
    [void]$thread.AddArgument($dst_dir)
    [void]$thread.AddArgument($_)
    [void]$threads.Add($thread)
    [void]$results.Add($thread.BeginInvoke())
}
'-------------------------------'

'等待线程结束'
while($true){
    $all_done = $true
    for($i = 0; $i -lt $results.Count; $i++){
        if($results[$i] -ne $null){
            if($results[$i].IsCompleted){
                $threads[$i].EndInvoke($results[$i])
				$threads[$i].Dispose()
				$threads[$i] = $null
                $results[$i] = $null
				[System.GC]::Collect()
            } else {
                $all_done = $false
            }
        }
    }
    if($all_done){ break }
    Start-Sleep -Milliseconds 500
}

#关闭线程池
$pool.Close()
'-------------------'
'{0}  -> {1}' -f $t1,(Get-Date)
复制代码

newswan

一段时间不用，很多都不熟悉了啊

newswan

回复 42# idwma


谢谢，这个用时是 unix 工具的 6 倍，比前面的要好多了

idwma

回复 40# newswan

去重的部分抄前辈的试试看快不快http://www.bathome.net/thread-25194-2-1.html

Get-ChildItem -path $dest *.tmp | foreach-object {
    $reader = New-Object -TypeName System.IO.StreamReader -ArgumentList $_.fullname
    $aa=New-Object System.Collections.Generic.HashSet[string]
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".txt")
    while ( $read = $reader.ReadLine() ) {
        out-null -InputObject $aa.add($read)
    }
    [IO.File]::WriteAllLines($filename,$aa)
}
复制代码

PCL0769

回复 31# xczxczxcz
老师好！能将这个PS发出来吗？谢谢！

newswan

回复 39# idwma


谢谢指点
net 没学过。。。
合并效率一样了，去重，还是很慢

$sour = "D:\share\tech\New folder\新建文件夹"
$dest = "D:\share\tech\New folder\ccc2"
$exclude = "QQ"

Remove-Item $dest\*.*

( $MyInvocation.MyCommand.Name + "  1  " + (get-date -Format "HH:mm:ss.ff").tostring() ) | out-file -Encoding ascii -append time.txt

Get-ChildItem -path $sour *.txt -Recurse | foreach-object {
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".tmp")
    $a=( [IO.File]::ReadAllLines($_.fullname) )
    $a = $a -match "\S+\s+\S+\s+\S+" -notmatch $exclude -replace "SZ","1" -replace "SH","0" -replace "\s+","|"
    [IO.File]::AppendAllLines([string]$filename , [string[]]$a)
}

( $MyInvocation.MyCommand.Name + "  2  " + (get-date -Format "HH:mm:ss.ff").tostring() ) | out-file -Encoding ascii -append time.txt

Get-ChildItem -path $dest *.tmp | foreach-object {
    $_.fullname
    $filename = Join-Path -path $dest -ChildPath ($_.basename + ".txt")
    $ht = @{}
    $a = ( [IO.File]::ReadAllLines($_.fullname) )
    $a = $a | foreach-object {
        if ( -not ( $ht.ContainsKey($_) ) )
        {
            $_
            $ht.add($_,"1")
        }
    }
    $a | out-file -Encoding utf8 $filename
}

( $MyInvocation.MyCommand.Name + "  3  " + (get-date -Format "HH:mm:ss.ff").tostring() ) | out-file -Encoding ascii -append time.txt
复制代码