Skip to content

Commit 82085a6

Browse files
committed
Replace gzip usage with open()
1 parent 099f957 commit 82085a6

File tree

1 file changed

+7
-6
lines changed

1 file changed

+7
-6
lines changed

tdclient/job_api.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#!/usr/bin/env python
22

33
import codecs
4-
import gzip
54
import json
65
import logging
76
import os
@@ -260,9 +259,9 @@ def job_result_format_each(
260259
raise ValueError("store_tmpfile works only when format is msgpack")
261260

262261
with tempfile.TemporaryDirectory() as tempdir:
263-
path = os.path.join(tempdir, f"{job_id}.msgpack.gz")
262+
path = os.path.join(tempdir, f"{job_id}.msgpack")
264263
self.download_job_result(job_id, path, num_threads)
265-
with gzip.GzipFile(path, "rb") as f:
264+
with open(path, "rb") as f:
266265
unpacker = msgpack.Unpacker(
267266
f, raw=False, max_buffer_size=1000 * 1024**2
268267
)
@@ -309,7 +308,7 @@ def download_job_result(self, job_id, path, num_threads=4):
309308
url = create_url(
310309
"/v3/job/result/{job_id}?format={format}",
311310
job_id=job_id,
312-
format="msgpack.gz",
311+
format="msgpack",
313312
)
314313

315314
def get_chunk(url, start, end):
@@ -322,8 +321,10 @@ def download_chunk(url, start, end, index, file_name):
322321
with get_chunk(url, start, end) as response:
323322
if response.status == 206: # Partial content (range supported)
324323
with open(f"{file_name}.part{index}", "wb") as f:
325-
for chunk in response.stream(1024):
326-
f.write(chunk)
324+
for chunk in response.stream(128 * 1024):
325+
if chunk:
326+
f.write(chunk)
327+
327328
return True
328329
else:
329330
log.warning(

0 commit comments

Comments
 (0)