From 512416ce2407de48a4449d33137829c959e2531e Mon Sep 17 00:00:00 2001 From: Jose Ortiz Date: Mon, 15 Oct 2018 19:23:24 -0300 Subject: [PATCH 1/2] Adds support for dataset reading (iter_rows) on Python 3 --- dataikuapi/utils.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/dataikuapi/utils.py b/dataikuapi/utils.py index e104eb54..f79367fe 100644 --- a/dataikuapi/utils.py +++ b/dataikuapi/utils.py @@ -5,6 +5,8 @@ import itertools if sys.version_info > (3,0): + import codecs + dku_basestring_type = str dku_zip_longest = itertools.zip_longest else: @@ -31,7 +33,7 @@ def next(self): def __iter__(self): return self - + def none_if_throws(f): def aux(*args, **kargs): try: @@ -51,7 +53,10 @@ def __init__(self, schema, csv_stream): def iter_rows(self): def decode(x): - return unicode(x, "utf8") + if sys.version_info > (3,0): + return x + else: + return unicode(x, "utf8") def parse_iso_date(s): if s == "": @@ -79,9 +84,17 @@ def str_to_bool(s): CASTERS.get(col["type"], decode) for col in schema ] with closing(self.csv_stream) as r: - for uncasted_tuple in csv.reader(r.raw, - delimiter='\t', - quotechar='"', - doublequote=True): - yield [none_if_throws(caster)(val) - for (caster, val) in dku_zip_longest(casters, uncasted_tuple)] + if sys.version_info > (3,0): + for uncasted_tuple in csv.reader(codecs.iterdecode(r.raw, 'utf-8'), + delimiter='\t', + quotechar='"', + doublequote=True): + yield [none_if_throws(caster)(val) + for (caster, val) in dku_zip_longest(casters, uncasted_tuple)] + else: + for uncasted_tuple in csv.reader(r.raw, + delimiter='\t', + quotechar='"', + doublequote=True): + yield [none_if_throws(caster)(val) + for (caster, val) in dku_zip_longest(casters, uncasted_tuple)] From ca2fa8fc1c5d70b826e20ee67086a8d3a350f57f Mon Sep 17 00:00:00 2001 From: Nicolas Servel Date: Thu, 8 Nov 2018 12:10:13 +0100 Subject: [PATCH 2/2] Refactor way of building raw generator depending on version of python --- dataikuapi/utils.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/dataikuapi/utils.py b/dataikuapi/utils.py index f79367fe..74f15380 100644 --- a/dataikuapi/utils.py +++ b/dataikuapi/utils.py @@ -85,16 +85,12 @@ def str_to_bool(s): ] with closing(self.csv_stream) as r: if sys.version_info > (3,0): - for uncasted_tuple in csv.reader(codecs.iterdecode(r.raw, 'utf-8'), - delimiter='\t', - quotechar='"', - doublequote=True): - yield [none_if_throws(caster)(val) - for (caster, val) in dku_zip_longest(casters, uncasted_tuple)] + raw_generator = codecs.iterdecode(r.raw, 'utf-8') else: - for uncasted_tuple in csv.reader(r.raw, - delimiter='\t', - quotechar='"', - doublequote=True): - yield [none_if_throws(caster)(val) - for (caster, val) in dku_zip_longest(casters, uncasted_tuple)] + raw_generator = r.raw + for uncasted_tuple in csv.reader(raw_generator, + delimiter='\t', + quotechar='"', + doublequote=True): + yield [none_if_throws(caster)(val) + for (caster, val) in dku_zip_longest(casters, uncasted_tuple)]