Parallel downloading and writing to stdout

This commit is contained in:
Egor Aristov 2020-07-18 20:05:46 +03:00
parent 57cd5f6ac6
commit 09463f491d

View File

@ -221,18 +221,28 @@ async def download_part(client: TelegramClient, dest: typing.BinaryIO, dInfo: Do
totalBytesDownloaded = 0 totalBytesDownloaded = 0
lastRealTimeMeasurement = time.time() lastRealTimeMeasurement = time.time()
buffer = None
currentLoop = asyncio.get_running_loop()
currentExecutor = ThreadPoolExecutor(max_workers=1)
def writeBuffer(buf):
if buf:
dest.write(buf)
while totalBytesDownloaded < realSize: while totalBytesDownloaded < realSize:
offset = totalBytesDownloaded offset = totalBytesDownloaded
limit = chunkSize limit = chunkSize
try: try:
downloadResult: File = await client(GetFileRequest( taskReceive = client(GetFileRequest(
inputFileLocation, inputFileLocation,
offset, offset,
limit, limit,
precise=False, precise=False,
cdn_supported=False cdn_supported=False
)) ))
taskWriteBuffer = currentLoop.run_in_executor(currentExecutor, writeBuffer, buffer)
downloadResult, writeResult = await asyncio.gather(taskReceive, taskWriteBuffer, loop=currentLoop, return_exceptions=False)
except BadRequestError as e: except BadRequestError as e:
if 'expire' in str(e.message).lower(): if 'expire' in str(e.message).lower():
click.echo('Reloading message...', err=True) click.echo('Reloading message...', err=True)
@ -249,7 +259,6 @@ async def download_part(client: TelegramClient, dest: typing.BinaryIO, dInfo: Do
extraBytes = totalBytesDownloaded - realSize extraBytes = totalBytesDownloaded - realSize
buffer = buffer[:-extraBytes] buffer = buffer[:-extraBytes]
dest.write(buffer)
if totalBytesDownloaded % (pow(2, 20) * 10) == 0: if totalBytesDownloaded % (pow(2, 20) * 10) == 0:
click.echo( click.echo(
f"{dInfo.part_info['part']}p {format_file_size(totalBytesDownloaded)}b {(time.time() - lastRealTimeMeasurement):.3f}s\t", f"{dInfo.part_info['part']}p {format_file_size(totalBytesDownloaded)}b {(time.time() - lastRealTimeMeasurement):.3f}s\t",