Compare commits

..

2 Commits

Author SHA1 Message Date
247128cbe9 Fix 2020-07-19 09:44:13 +03:00
09463f491d Parallel downloading and writing to stdout 2020-07-18 20:05:46 +03:00

View File

@ -221,18 +221,28 @@ async def download_part(client: TelegramClient, dest: typing.BinaryIO, dInfo: Do
totalBytesDownloaded = 0 totalBytesDownloaded = 0
lastRealTimeMeasurement = time.time() lastRealTimeMeasurement = time.time()
buffer = None
currentLoop = asyncio.get_running_loop()
currentExecutor = ThreadPoolExecutor(max_workers=1)
def writeBuffer(buf):
if buf:
dest.write(buf)
while totalBytesDownloaded < realSize: while totalBytesDownloaded < realSize:
offset = totalBytesDownloaded offset = totalBytesDownloaded
limit = chunkSize limit = chunkSize
try: try:
downloadResult: File = await client(GetFileRequest( taskReceive = client(GetFileRequest(
inputFileLocation, inputFileLocation,
offset, offset,
limit, limit,
precise=False, precise=False,
cdn_supported=False cdn_supported=False
)) ))
taskWriteBuffer = currentLoop.run_in_executor(currentExecutor, writeBuffer, buffer)
downloadResult, writeResult = await asyncio.gather(taskReceive, taskWriteBuffer, loop=currentLoop, return_exceptions=False)
except BadRequestError as e: except BadRequestError as e:
if 'expire' in str(e.message).lower(): if 'expire' in str(e.message).lower():
click.echo('Reloading message...', err=True) click.echo('Reloading message...', err=True)
@ -249,12 +259,12 @@ async def download_part(client: TelegramClient, dest: typing.BinaryIO, dInfo: Do
extraBytes = totalBytesDownloaded - realSize extraBytes = totalBytesDownloaded - realSize
buffer = buffer[:-extraBytes] buffer = buffer[:-extraBytes]
dest.write(buffer)
if totalBytesDownloaded % (pow(2, 20) * 10) == 0: if totalBytesDownloaded % (pow(2, 20) * 10) == 0:
click.echo( click.echo(
f"{dInfo.part_info['part']}p {format_file_size(totalBytesDownloaded)}b {(time.time() - lastRealTimeMeasurement):.3f}s\t", f"{dInfo.part_info['part']}p {format_file_size(totalBytesDownloaded)}b {(time.time() - lastRealTimeMeasurement):.3f}s\t",
nl=False, err=True nl=False, err=True
) )
writeBuffer(buffer)
async def download_file(client: TelegramClient, dest: typing.BinaryIO, fileInfo: dict, dialog: str): async def download_file(client: TelegramClient, dest: typing.BinaryIO, fileInfo: dict, dialog: str):